1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
#!/usr/bin/env python
import os
import sys
import re
import difflib
from ConfigParser import ConfigParser
from StringIO import StringIO
import psycopg2
sys.path.append('..')
from lib.storage import ArchivesParserStorage
if __name__ == "__main__":
cfg = ConfigParser()
cfg.read('%s/../archives.ini' % os.path.realpath(os.path.dirname(sys.argv[0])))
try:
connstr = cfg.get('db', 'connstr')
except Exception:
connstr = 'need_connstr'
conn = psycopg2.connect(connstr)
curs = conn.cursor()
with open('fromlist', 'r') as f:
for l in f:
curs.execute("SAVEPOINT msg")
msgid = l.strip()
curs.execute("SELECT id, rawtxt, bodytxt FROM messages WHERE messageid=%(msgid)s", {
'msgid': msgid,
})
id, rawtxt, bodytxt = curs.fetchone()
ap = ArchivesParserStorage()
s = StringIO(rawtxt)
# Parse the old message, so we can fix it.
ap.parse(s)
ap.analyze()
# Double check...
if bodytxt.decode('utf8') == ap.bodytxt:
print "Message already fixed: %s" % msgid
curs.execute("ROLLBACK TO SAVEPOINT msg")
continue
# Now try to fix it...
s.seek(0)
fixed = re.sub('^>From ', 'From ', s.getvalue(), flags=re.MULTILINE)
curs.execute("UPDATE messages SET rawtxt=%(raw)s WHERE messageid=%(msgid)s", {
'msgid': msgid,
'raw': bytearray(fixed),
})
# Ok, read it back and try again
curs.execute("SELECT id, rawtxt, bodytxt FROM messages WHERE messageid=%(msgid)s", {
'msgid': msgid,
})
id, rawtxt, bodytxt = curs.fetchone()
ap = ArchivesParserStorage()
# Parse the old message, so we can
ap.parse(StringIO(rawtxt))
ap.analyze()
if ap.bodytxt != bodytxt.decode('utf8'):
print "Failed to fix %s!" % msgid
# Generate diff to show what we changed
print "CHANGED:"
print "\n".join(difflib.unified_diff(s.getvalue(),
fixed,
fromfile='old',
tofile='new',
n=2,
lineterm=''))
print "----"
# Generate a diff to show what's left
print "REMAINING:"
print "\n".join(difflib.unified_diff(bodytxt.decode('utf8').splitlines(),
ap.bodytxt.splitlines(),
fromfile='old',
tofile='new',
n=2,
lineterm=''))
print "--------------"
while True:
a = raw_input('Save this change anyway?').lower()
if a == 'y' or a == 'yes':
print "Ok, saving!"
curs.execute("RELEASE SAVEPOINT msg")
break
elif a == 'n' or a == 'no':
print "Ok, rolling back!"
curs.execute("ROLLBACK TO SAVEPOINT msg")
break
elif a == 'yq':
print "Ok, committing and then exiting"
curs.execute("RELEASE SAVEPOINT msg")
conn.commit()
conn.close()
sys.exit(0)
else:
print "Fixed %s!" % msgid
curs.execute("RELEASE SAVEPOINT msg")
s.close()
print "Committing all that's there..."
conn.commit()
conn.close()
|
