summaryrefslogtreecommitdiff
path: root/loader/tools/fix_from.py
blob: 52efec9b4a25a7566823f9933d57191be97ae81b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#!/usr/bin/env python

import os
import sys
import re
import difflib

from ConfigParser import ConfigParser
from StringIO import StringIO

import psycopg2

sys.path.append('..')
from lib.storage import ArchivesParserStorage

if __name__ == "__main__":
    cfg = ConfigParser()
    cfg.read('%s/../archives.ini' % os.path.realpath(os.path.dirname(sys.argv[0])))
    try:
        connstr = cfg.get('db', 'connstr')
    except Exception:
        connstr = 'need_connstr'

    conn = psycopg2.connect(connstr)
    curs = conn.cursor()

    with open('fromlist', 'r') as f:
        for l in f:
            curs.execute("SAVEPOINT msg")

            msgid = l.strip()
            curs.execute("SELECT id, rawtxt, bodytxt FROM messages WHERE messageid=%(msgid)s", {
                'msgid': msgid,
            })
            id, rawtxt, bodytxt = curs.fetchone()

            ap = ArchivesParserStorage()
            s = StringIO(rawtxt)

            # Parse the old message, so we can fix it.
            ap.parse(s)
            ap.analyze()

            # Double check...
            if bodytxt.decode('utf8') == ap.bodytxt:
                print "Message already fixed: %s" % msgid
                curs.execute("ROLLBACK TO SAVEPOINT msg")
                continue

            # Now try to fix it...
            s.seek(0)

            fixed = re.sub('^>From ', 'From ', s.getvalue(), flags=re.MULTILINE)

            curs.execute("UPDATE messages SET rawtxt=%(raw)s WHERE messageid=%(msgid)s", {
                'msgid': msgid,
                'raw': bytearray(fixed),
            })

            # Ok, read it back and try again
            curs.execute("SELECT id, rawtxt, bodytxt FROM messages WHERE messageid=%(msgid)s", {
                'msgid': msgid,
            })
            id, rawtxt, bodytxt = curs.fetchone()

            ap = ArchivesParserStorage()

            # Parse the old message, so we can
            ap.parse(StringIO(rawtxt))
            ap.analyze()

            if ap.bodytxt != bodytxt.decode('utf8'):
                print "Failed to fix %s!" % msgid

                # Generate diff to show what we changed
                print "CHANGED:"
                print "\n".join(difflib.unified_diff(s.getvalue(),
                                                     fixed,
                                                     fromfile='old',
                                                     tofile='new',
                                                     n=2,
                                                     lineterm=''))
                print "----"
                # Generate a diff to show what's left
                print "REMAINING:"
                print "\n".join(difflib.unified_diff(bodytxt.decode('utf8').splitlines(),
                                                     ap.bodytxt.splitlines(),
                                                     fromfile='old',
                                                     tofile='new',
                                                     n=2,
                                                     lineterm=''))
                print "--------------"
                while True:
                    a = raw_input('Save this change anyway?').lower()
                    if a == 'y' or a == 'yes':
                        print "Ok, saving!"
                        curs.execute("RELEASE SAVEPOINT msg")
                        break
                    elif a == 'n' or a == 'no':
                        print "Ok, rolling back!"
                        curs.execute("ROLLBACK TO SAVEPOINT msg")
                        break
                    elif a == 'yq':
                        print "Ok, committing and then exiting"
                        curs.execute("RELEASE SAVEPOINT msg")
                        conn.commit()
                        conn.close()
                        sys.exit(0)
            else:
                print "Fixed %s!" % msgid
                curs.execute("RELEASE SAVEPOINT msg")
            s.close()

    print "Committing all that's there..."
    conn.commit()
    conn.close()