1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
#!/usr/bin/python3
# Copyright (c) 2009-2013, 2020 Christoph Berg <myon@debian.org>
#
# getdpkginfo by
# Copyright (C) 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 3. The name of the author may not be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import apt_pkg, psycopg2, os, re, sys, time
pg = psycopg2.connect('service=pgapt')
cur = pg.cursor()
def parseSourceFile(distribution, component, packagesfile, ctime):
if not os.path.isfile(packagesfile):
raise Exception("%s not found" % packagesfile)
stdout = os.popen("bzcat '%s'" % packagesfile)
#stdout = file(packagesfile)
parse = apt_pkg.TagFile(stdout)
# clear the suite
cur.execute("BEGIN")
cur.execute("""DELETE FROM sourcelist WHERE
(distribution, component) = (%s, %s)""",
[distribution, component])
# for every package ...
while parse.step():
package = parse.section.get('Package')
version = parse.section.get('Version')
control = "\n".join([ "%s: %s" % (k, parse.section.get(k)) \
for k in parse.section.keys() ])
# store package control file
cur.execute("""SELECT 1 FROM source
WHERE (source, srcversion) = (%s, %s)""",
[package, version])
found = cur.fetchone()
if not found: # this is the first time we see this package
cur.execute("""INSERT INTO src (source) VALUES (%s)
ON CONFLICT (source)
DO UPDATE SET upload = NULL""",
[package])
cur.execute("""INSERT INTO source (source, srcversion, control, c, time)
VALUES (%s, %s, %s, control2jsonb(%s), %s)""",
[package, version, control, control, ctime])
cur.execute("""INSERT INTO sourcefile (source, srcversion, directory, filename)
SELECT %s, %s, c->>'directory', regexp_replace(f, '.* ', '')
FROM control2jsonb(%s) c, jsonb_array_elements_text(c->'files') f
ON CONFLICT DO NOTHING""",
[package, version, control])
# finally, add the package to the suite's package list
cur.execute("""INSERT INTO sourcelist
(distribution, component, source, srcversion)
VALUES (%s, %s, %s, %s)""",
[distribution, component, package, version])
# record info in suite history table
cur.execute("""INSERT INTO sourcehist
(distribution, component, source, srcversion, time)
SELECT distribution, component, source, srcversion, %s
FROM sourcelist l
WHERE (distribution, component) = (%s, %s) AND NOT EXISTS
(SELECT * FROM sourcehist h WHERE
(l.distribution, l.component, l.source, l.srcversion) =
(h.distribution, h.component, h.source, h.srcversion))""",
[ctime, distribution, component])
(packagesfile, distribution, component) = (sys.argv[1], sys.argv[2], sys.argv[3])
if not distribution in packagesfile:
raise NameError("distribution is not part of filename")
if not component in packagesfile:
raise NameError("component is not part of filename")
verbose = len(sys.argv) > 4
cur.execute("""SELECT extract (epoch from last_update) AS last_update
FROM srcdistribution
WHERE (distribution, component) = (%s, %s)
FOR UPDATE NOWAIT""", [distribution, component])
last_update = cur.fetchone()
mtime = os.path.getmtime(packagesfile)
ctime = time.ctime(mtime)
if not last_update or mtime > last_update[0]:
if verbose:
print("Reading %s" % packagesfile)
update = """INSERT INTO srcdistribution (distribution, component, last_update) VALUES (%s, %s, %s)
ON CONFLICT (distribution, component) DO UPDATE SET last_update = %s"""
cur.execute(update, [distribution, component, ctime, ctime])
parseSourceFile(distribution, component, packagesfile, ctime)
cur.execute("COMMIT")
else:
if verbose:
print("Skipping up to date %s" % packagesfile)
# Insert missing source packages:
# INSERT INTO package_source SELECT p.package_id, s.package_id as source_id from package p JOIN package s ON (p.package = s.package AND p.version = s.version AND s.pkg_architecture='source') where p.package_id not in (select package_id from package_source ) and p.pkg_architecture <> 'source';
# Insert missing source packages, packages with broken/missing binnmu Source: fieds:
# INSERT INTO package_source SELECT p.package_id, s.package_id as source_id from package p JOIN package s ON (p.package = s.package AND regexp_replace (p.version, E'\\+b\\d+$', '') = s.version AND s.pkg_architecture='source') where p.package_id not in (select package_id from package_source ) and p.pkg_architecture <> 'source';
|
