The program attached to this message makes the Python interpreter segfault
randomly. I have tried both Python 2.2 which came with Debian Stable, and
self-compiled Python 2.3.3 (newest I could find on www.python.org,
compiled with default options (./configure && make). I'm using the pyPgSQL
plugin to connect to a PostGreSQL database, and have tried the Debian and
self-compiled newest versions of that as well.
I'm running BitTorrent, and that works perfectly well; btlaunchmany.py has
been running for months continuously without any problems. I've also run
the kernel compile test (compiling the Linux kernel nonstop to find any
inadequeties in processor cooling), and couldn't get any errors in 6 hours.
This makes me thing I'm hitting some weird bug in the interpreter.
Specifically, I'm wondering if my habit of reusing old variable names in a
function once they are no longer needed might be causing the trouble;
maybe it causes confusion on the variable type ?
The program retrieves Usenet News messages from the database (inserted
there by another Python program, which works perfectly and also uses the
pyPgSQL plugin).
So, here's the program. Does anyone know what's wrong with it ?
#!/usr/local/bin/python2.3
# Insert message contents into the database, for each message-id already there
#
# Copyright 2004 by Juho Saarikko
# License: GNU General Public License (GPL) version 2
# See www.gnu.org for details
from pyPgSQL import libpq
import nntplib
import sys
import string
import regex
import sha
import imghdr
import binascii
import StringIO
import os
def strip_trailing_ dots(n):
tmp = []
for i in range(len(n)):
if n[i][-1] == "," or n[i][-1] == ".":
tmp.append(n[i][:-1])
else:
tmp.append(n[i])
return tmp
def findmimetype(bo dy, filename):
what = imghdr.what(Str ingIO.StringIO( body))
if what == "gif":
return "image/gif"
if what == "png":
return "image/png"
if what == "jpeg":
return "image/jpeg"
return None
def try_decode_and_ insert_uuencode d(conn, id):
begin = regex.compile(" begin [0-9]+ \(.*\)")
conn.query("BEG IN")
basedir = "kuvat"
message = conn.query("SEL ECT data FROM fragments_bodie s WHERE message = " + str(id) + " ORDER BY line")
print message.ntuples
keywords = []
picids = []
n = 0
s = ""
print 'Starting message id ' + str(id)
while n < message.ntuples :
# print "length of row " + str(n)
# print str(message.get length(n, 0))
# print "Got length"
s = str(message.get value(n, 0))
# print "Got s"
if begin.match(s) > 0:
# print "Begin matched"
body = []
file = begin.group(1)
# print "Starting to decode, at line " + str(n + 1)
for k in range(n+1, message.ntuples ):
# print "Decodind row " + str(k)
s = message.getvalu e(k, 0)
if s[:3] == "end":
n = k + 1
break
try:
body.append(bin ascii.a2b_uu(li bpq.PgUnquoteBy tea(s)))
except:
bytes = (((ord(s[0])-32) & 63) * 4 + 3) / 3
body.append(bin ascii.a2b_uu(s[:bytes]))
# print "Got to end, at line " + str(n)
# print "Attempting to join body"
body = string.join(bod y, "")
# print "Attempting to hash body"
hash = sha.new(body)
qhash = libpq.PgQuoteBy tea(hash.digest ())
# qbody = libpq.PgQuoteBy tea(body)
# print "Attempting to find whether the pic already exists"
already = conn.query("SEL ECT id FROM pictures WHERE hash = " + qhash)
if already.ntuples == 0:
# print "Attempting to find mimetype"
mimetype = findmimetype(bo dy, file)
# print "Found mimetype"
if mimetype != None:
# o = conn.query("INS ERT INTO pictures (picture, hash, mimetype) VALUES (" + qbody + ", " + qhash + ", " + libpq.PgQuoteSt ring(mimetype) + ")")
# already = conn.query("SEL ECT id FROM pictures WHERE OID = " + str(o.oidValue( )));
# already = conn.query("SEL ECT id FROM pictures WHERE data = " + qbody)
# already = conn.query("SEL ECT id FROM pictures WHERE hash = " + qhash)
# print "Attempting to insert hash and mimetype"
conn.query("INS ERT INTO pictures (hash, mimetype) VALUES (" + qhash + ", " + libpq.PgQuoteSt ring(mimetype) + ")")
# print "Attempting to get id"
already = conn.query("SEL ECT id FROM pictures WHERE hash = " + qhash)
# print "Attempting to get value"
picid = already.getvalu e(0, 0)
# print "Attempting to OK dir"
if os.access(based ir + "/tmp", os.F_OK) != 1:
os.mkdir(basedi r + "/tmp")
fh = open(basedir + "/tmp/" + str(picid), "wb")
fh.write(body)
fh.close()
# print "File ok"
else:
picid = already.getvalu e(0, 0)
if already.ntuples == 0:
# print "already.ntuple s == 0, ROLLBACKing"
conn.query("ROL LBACK")
return
# print "Appending picid"
picids.append(p icid)
# print "Picid appended"
else:
tmpkey = strip_trailing_ dots(string.spl it(s))
if len(tmpkey) > 0:
for j in range(len(tmpke y)):
keywords.append (tmpkey[j])
# print "Adding 1 to n"
n = n + 1
if len(picids) > 0:
# print "Finding Subject"
head = conn.query("SEL ECT contents FROM fragments_heade r_contents WHERE message = " + str(id) + " AND header = (SELECT id FROM fragments_heade r_names WHERE header ilike 'Subject')")
if head.ntuples > 0:
# print "Splitting Subject"
blah = head.getvalue(0 ,0)
print str(blah)
blahblah = string.split(st r(blah))
# print "Stripping"
abctmpkey = strip_trailing_ dots(blahblah)
# print "Stripping done"
# print "Really"
tmpkey = abctmpkey
#B print "Subject split"
if len(tmpkey) > 0:
for j in range(len(tmpke y)):
keywords.append (tmpkey[j])
o = conn.query("INS ERT INTO messages DEFAULT VALUES")
mid = conn.query("SEL ECT id FROM messages WHERE OID = " + str(o.oidValue) )
messageid = mid.getvalue(0, 0)
if len(keywords) > 0:
for x in range(len(tmpke y)):
qword = libpq.PgQuoteSt ring(str(keywor ds[x]))
tmp = conn.query("SEL ECT id FROM keywords_words WHERE keyword = " + qword)
if tmp.ntuples == 0:
conn.query("INS ERT INTO keywords_words (keyword) VALUES (" + qword + ")")
tmp = conn.query("SEL ECT id FROM keywords_words WHERE keyword = " + qword)
keyid = str(tmp.getvalu e(0, 0))
for y in range(len(picid s)):
conn.query("INS ERT INTO keywords_glue(w ord, picture) VALUES (" + keyid + ", " + str(picids[y]) + ")")
dummyone = "SELECT fragments_heade r_contents.line , fragments_heade r_names.header, "
dummytwo = " fragments_heade r_contents.cont ents FROM fragments_heade r_names, fragments_heade r_contents"
dummythree = " WHERE fragments_heade r_contents.mess age = " + str(id)
dummyfour = " AND fragments_heade r_contents.head er = fragments_heade r_names.id"
head = conn.query(dumm yone + dummytwo + dummythree + dummyfour)
if head.ntuples > 0:
for h in range(head.ntup les):
qhead = libpq.PgQuoteSt ring(str(head.g etvalue(h, 1)))
qcont = libpq.PgQuoteSt ring(str(head.g etvalue(h, 2)))
tmp = conn.query("SEL ECT id FROM header_names WHERE header = " + qhead)
if tmp.ntuples == 0:
conn.query("INS ERT INTO header_names (header) VALUES (" + qhead + ")")
tmp = conn.query("SEL ECT id FROM header_names WHERE header = " + qhead)
headid = str(tmp.getvalu e(0, 0))
line = str(head.getval ue(0, 0))
conn.query("INS ERT INTO header_contents (header, message, line, contents) VALUES (" + headid + ", " + str(messageid) + ", " + line + ", " + qcont + ")")
conn.query("DEL ETE FROM fragments_heade r_contents WHERE message = " + str(id))
conn.query("DEL ETE FROM fragments_bodie s WHERE message = " + str(id))
conn.query("COM MIT")
tmpdir = basedir + "/tmp/"
for i in range(len(picid s)):
picid = picids[i]
if os.access(based ir + "/" + str(picid%1000) , os.F_OK) != 1:
os.mkdir(basedi r + "/" + str(picid%1000) )
os.link(tmpdir + str(picid), basedir + "/" + str(picid%1000) + "/" + str(picid))
os.unlink(tmpdi r +str(picid))
else:
conn.query("ROL LBACK")
return
database = libpq.PQconnect db('dbname = kuvat')
items = database.query( "SELECT message FROM whole_attachmen ts")
# try_decode_and_ insert_uuencode d(database, 1167)
for i in range(items.ntu ples):
print 'Starting call ' + str(i)
try_decode_and_ insert_uuencode d(database, items.getvalue( items.ntuples - 1 - i,0))
print ' returned from call ' + str(i)
# except:
# print 'Some other error occurred, trying to continue...\n'
randomly. I have tried both Python 2.2 which came with Debian Stable, and
self-compiled Python 2.3.3 (newest I could find on www.python.org,
compiled with default options (./configure && make). I'm using the pyPgSQL
plugin to connect to a PostGreSQL database, and have tried the Debian and
self-compiled newest versions of that as well.
I'm running BitTorrent, and that works perfectly well; btlaunchmany.py has
been running for months continuously without any problems. I've also run
the kernel compile test (compiling the Linux kernel nonstop to find any
inadequeties in processor cooling), and couldn't get any errors in 6 hours.
This makes me thing I'm hitting some weird bug in the interpreter.
Specifically, I'm wondering if my habit of reusing old variable names in a
function once they are no longer needed might be causing the trouble;
maybe it causes confusion on the variable type ?
The program retrieves Usenet News messages from the database (inserted
there by another Python program, which works perfectly and also uses the
pyPgSQL plugin).
So, here's the program. Does anyone know what's wrong with it ?
#!/usr/local/bin/python2.3
# Insert message contents into the database, for each message-id already there
#
# Copyright 2004 by Juho Saarikko
# License: GNU General Public License (GPL) version 2
# See www.gnu.org for details
from pyPgSQL import libpq
import nntplib
import sys
import string
import regex
import sha
import imghdr
import binascii
import StringIO
import os
def strip_trailing_ dots(n):
tmp = []
for i in range(len(n)):
if n[i][-1] == "," or n[i][-1] == ".":
tmp.append(n[i][:-1])
else:
tmp.append(n[i])
return tmp
def findmimetype(bo dy, filename):
what = imghdr.what(Str ingIO.StringIO( body))
if what == "gif":
return "image/gif"
if what == "png":
return "image/png"
if what == "jpeg":
return "image/jpeg"
return None
def try_decode_and_ insert_uuencode d(conn, id):
begin = regex.compile(" begin [0-9]+ \(.*\)")
conn.query("BEG IN")
basedir = "kuvat"
message = conn.query("SEL ECT data FROM fragments_bodie s WHERE message = " + str(id) + " ORDER BY line")
print message.ntuples
keywords = []
picids = []
n = 0
s = ""
print 'Starting message id ' + str(id)
while n < message.ntuples :
# print "length of row " + str(n)
# print str(message.get length(n, 0))
# print "Got length"
s = str(message.get value(n, 0))
# print "Got s"
if begin.match(s) > 0:
# print "Begin matched"
body = []
file = begin.group(1)
# print "Starting to decode, at line " + str(n + 1)
for k in range(n+1, message.ntuples ):
# print "Decodind row " + str(k)
s = message.getvalu e(k, 0)
if s[:3] == "end":
n = k + 1
break
try:
body.append(bin ascii.a2b_uu(li bpq.PgUnquoteBy tea(s)))
except:
bytes = (((ord(s[0])-32) & 63) * 4 + 3) / 3
body.append(bin ascii.a2b_uu(s[:bytes]))
# print "Got to end, at line " + str(n)
# print "Attempting to join body"
body = string.join(bod y, "")
# print "Attempting to hash body"
hash = sha.new(body)
qhash = libpq.PgQuoteBy tea(hash.digest ())
# qbody = libpq.PgQuoteBy tea(body)
# print "Attempting to find whether the pic already exists"
already = conn.query("SEL ECT id FROM pictures WHERE hash = " + qhash)
if already.ntuples == 0:
# print "Attempting to find mimetype"
mimetype = findmimetype(bo dy, file)
# print "Found mimetype"
if mimetype != None:
# o = conn.query("INS ERT INTO pictures (picture, hash, mimetype) VALUES (" + qbody + ", " + qhash + ", " + libpq.PgQuoteSt ring(mimetype) + ")")
# already = conn.query("SEL ECT id FROM pictures WHERE OID = " + str(o.oidValue( )));
# already = conn.query("SEL ECT id FROM pictures WHERE data = " + qbody)
# already = conn.query("SEL ECT id FROM pictures WHERE hash = " + qhash)
# print "Attempting to insert hash and mimetype"
conn.query("INS ERT INTO pictures (hash, mimetype) VALUES (" + qhash + ", " + libpq.PgQuoteSt ring(mimetype) + ")")
# print "Attempting to get id"
already = conn.query("SEL ECT id FROM pictures WHERE hash = " + qhash)
# print "Attempting to get value"
picid = already.getvalu e(0, 0)
# print "Attempting to OK dir"
if os.access(based ir + "/tmp", os.F_OK) != 1:
os.mkdir(basedi r + "/tmp")
fh = open(basedir + "/tmp/" + str(picid), "wb")
fh.write(body)
fh.close()
# print "File ok"
else:
picid = already.getvalu e(0, 0)
if already.ntuples == 0:
# print "already.ntuple s == 0, ROLLBACKing"
conn.query("ROL LBACK")
return
# print "Appending picid"
picids.append(p icid)
# print "Picid appended"
else:
tmpkey = strip_trailing_ dots(string.spl it(s))
if len(tmpkey) > 0:
for j in range(len(tmpke y)):
keywords.append (tmpkey[j])
# print "Adding 1 to n"
n = n + 1
if len(picids) > 0:
# print "Finding Subject"
head = conn.query("SEL ECT contents FROM fragments_heade r_contents WHERE message = " + str(id) + " AND header = (SELECT id FROM fragments_heade r_names WHERE header ilike 'Subject')")
if head.ntuples > 0:
# print "Splitting Subject"
blah = head.getvalue(0 ,0)
print str(blah)
blahblah = string.split(st r(blah))
# print "Stripping"
abctmpkey = strip_trailing_ dots(blahblah)
# print "Stripping done"
# print "Really"
tmpkey = abctmpkey
#B print "Subject split"
if len(tmpkey) > 0:
for j in range(len(tmpke y)):
keywords.append (tmpkey[j])
o = conn.query("INS ERT INTO messages DEFAULT VALUES")
mid = conn.query("SEL ECT id FROM messages WHERE OID = " + str(o.oidValue) )
messageid = mid.getvalue(0, 0)
if len(keywords) > 0:
for x in range(len(tmpke y)):
qword = libpq.PgQuoteSt ring(str(keywor ds[x]))
tmp = conn.query("SEL ECT id FROM keywords_words WHERE keyword = " + qword)
if tmp.ntuples == 0:
conn.query("INS ERT INTO keywords_words (keyword) VALUES (" + qword + ")")
tmp = conn.query("SEL ECT id FROM keywords_words WHERE keyword = " + qword)
keyid = str(tmp.getvalu e(0, 0))
for y in range(len(picid s)):
conn.query("INS ERT INTO keywords_glue(w ord, picture) VALUES (" + keyid + ", " + str(picids[y]) + ")")
dummyone = "SELECT fragments_heade r_contents.line , fragments_heade r_names.header, "
dummytwo = " fragments_heade r_contents.cont ents FROM fragments_heade r_names, fragments_heade r_contents"
dummythree = " WHERE fragments_heade r_contents.mess age = " + str(id)
dummyfour = " AND fragments_heade r_contents.head er = fragments_heade r_names.id"
head = conn.query(dumm yone + dummytwo + dummythree + dummyfour)
if head.ntuples > 0:
for h in range(head.ntup les):
qhead = libpq.PgQuoteSt ring(str(head.g etvalue(h, 1)))
qcont = libpq.PgQuoteSt ring(str(head.g etvalue(h, 2)))
tmp = conn.query("SEL ECT id FROM header_names WHERE header = " + qhead)
if tmp.ntuples == 0:
conn.query("INS ERT INTO header_names (header) VALUES (" + qhead + ")")
tmp = conn.query("SEL ECT id FROM header_names WHERE header = " + qhead)
headid = str(tmp.getvalu e(0, 0))
line = str(head.getval ue(0, 0))
conn.query("INS ERT INTO header_contents (header, message, line, contents) VALUES (" + headid + ", " + str(messageid) + ", " + line + ", " + qcont + ")")
conn.query("DEL ETE FROM fragments_heade r_contents WHERE message = " + str(id))
conn.query("DEL ETE FROM fragments_bodie s WHERE message = " + str(id))
conn.query("COM MIT")
tmpdir = basedir + "/tmp/"
for i in range(len(picid s)):
picid = picids[i]
if os.access(based ir + "/" + str(picid%1000) , os.F_OK) != 1:
os.mkdir(basedi r + "/" + str(picid%1000) )
os.link(tmpdir + str(picid), basedir + "/" + str(picid%1000) + "/" + str(picid))
os.unlink(tmpdi r +str(picid))
else:
conn.query("ROL LBACK")
return
database = libpq.PQconnect db('dbname = kuvat')
items = database.query( "SELECT message FROM whole_attachmen ts")
# try_decode_and_ insert_uuencode d(database, 1167)
for i in range(items.ntu ples):
print 'Starting call ' + str(i)
try_decode_and_ insert_uuencode d(database, items.getvalue( items.ntuples - 1 - i,0))
print ' returned from call ' + str(i)
# except:
# print 'Some other error occurred, trying to continue...\n'
Comment