1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
#!/bin/bash
set -eu
set -o pipefail
# S3 bucket
S3="s3://apt-archive.postgresql.org/pub/repos/apt"
PATH=$PATH:/srv/apt/repo/bin
cd /srv/apt/archive/pub/repos/apt
export PGSERVICE=pgapt
DIST_ARCHS="SELECT distinct distribution, architecture FROM distribution WHERE last_update > now() - '2 days'::interval ORDER BY 1, 2;"
DISTS="SELECT distinct distribution FROM srcdistribution WHERE last_update > now() - '2 days'::interval ORDER BY 1;"
# generate Packages files
psql -AXt -F ' ' -c "$DIST_ARCHS" | \
while read distribution architecture; do
[ -t 1 ] && echo "$distribution-archive/$architecture ..."
DIR="dists/$distribution-archive/main/binary-$architecture"
mkdir -p $DIR
PACKAGES="SELECT concat_ws(E'\n', format(E'%s\nTimestamp: %s\n', p.control, h.time)) FROM packagehist h JOIN package p ON (h.package, h.version, h.arch) = (p.package, p.version, p.arch) WHERE (h.distribution, h.architecture) = ('$distribution', '$architecture');"
psql -AXtc "$PACKAGES" > $DIR/Packages # generate both uncompressed and compressed version
rm -f $DIR/Packages.bz2
bzip2 --keep $DIR/Packages
done
# generate Sources and InRelease files
psql -AXt -F ' ' -c "$DISTS" | \
while read distribution; do
[ -t 1 ] && echo "$distribution-archive/source ..."
DIR="dists/$distribution-archive/main/source"
mkdir -p $DIR
PACKAGES="SELECT concat_ws(E'\n', format(E'%s\nTimestamp: %s\n', s.control, h.time)) FROM sourcehist h JOIN source s ON (h.source, h.srcversion) = (s.source, s.srcversion) WHERE h.distribution = '$distribution';"
psql -AXtc "$PACKAGES" > $DIR/Sources # generate both uncompressed and compressed version
rm -f $DIR/Sources.bz2
bzip2 --keep $DIR/Sources
# generate InRelease files for this distribution (includes Packages files from above)
DIR="dists/$distribution-archive"
(
cd $DIR
rm -f InRelease
RELEASE="$(apt-ftparchive release \
-o APT::FTPArchive::Release::Origin=apt-archive.postgresql.org \
-o APT::FTPArchive::Release::Label='PostgreSQL for Debian/Ubuntu archive repository' \
-o APT::FTPArchive::Release::Suite=$distribution-archive \
-o APT::FTPArchive::Release::Codename=$distribution-archive \
.)"
echo "$RELEASE" | gpg --clearsign > InRelease
# remove uncompressed files after they have been scanned
rm main/source/Sources main/*/Packages
)
indexhtml $DIR
indexhtml $DIR/main
for dir in $DIR/main/*/; do
indexhtml $dir
done
done
indexhtml dists
# upload to S3 and invalidate cache for the dists/ dir
aws s3 sync --quiet /srv/apt/archive/pub/repos/apt/dists $S3/dists
aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/dists/*" > /dev/null
cd /srv/apt/archive/pub/repos/apt
# generate index files for package pool directories
PACKAGES=$(psql -XAtc "SELECT source FROM src WHERE upload IS NULL")
for PACKAGE in $PACKAGES; do
case $PACKAGE in
lib*) INITIAL="$(echo $PACKAGE | cut -c 1-4)" ;;
*) INITIAL="$(echo $PACKAGE | cut -c 1)" ;;
esac
DIRECTORY="pool/main/$INITIAL/$PACKAGE"
QUERY="SELECT c->>'directory' || '/' || sf.filename AS path, sf.filename, NULL::text AS size, s.time
FROM source s
JOIN sourcefile sf ON (s.source, s.srcversion) = (sf.source, sf.srcversion)
WHERE s.source = '$PACKAGE'
UNION ALL SELECT p.c->>'filename', regexp_replace(p.c->>'filename', '.*/', ''), p.c->>'size', p.time
FROM package p
WHERE p.source = '$PACKAGE'
ORDER BY filename"
FORMAT="WITH q AS ($QUERY)
SELECT format(E'<a href=\"/pub/repos/apt/%s\">%s</a> %s %s <br /> \\n',
regexp_replace(path, '\\+', '%2B', 'g'),
filename,
time,
size || ' bytes')
FROM q"
mkdir -p "$DIRECTORY"
(
echo "<tt>"
echo "<a href=\"../index.html\">../</a> <br />"
psql -XAtc "$FORMAT"
echo "</tt>"
) > "$DIRECTORY/index.html"
upload-to-s3 "$DIRECTORY/index.html"
aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/$DIRECTORY/index.html" > /dev/null
indexhtml "pool/main/$INITIAL"
upload-to-s3 "pool/main/$INITIAL/index.html"
psql -Xqc "UPDATE src SET upload = true WHERE source = '$PACKAGE'"
done
# alert about any failed uploads
psql -XAt <<EOF
SELECT 'failed sourcefile upload:', source, srcversion FROM sourcefile WHERE upload IS FALSE;
SELECT 'failed package upload:', package, version, arch FROM package WHERE upload IS FALSE;
SELECT 'failed pool/main/p/package/index.html upload:', * FROM src WHERE upload IS FALSE;
EOF
|
