Add support for static deployment directly from a git branch

This avoids needing one repo cloned for each branch for returning
conferences (which is most of them).

1 files changed, 120 insertions, 22 deletions

diff --git a/tools/deploystatic/deploystatic.py b/tools/deploystatic/deploystatic.py
index e6704ae6..549dafb4 100755
--- a/tools/deploystatic/deploystatic.py
+++ b/tools/deploystatic/deploystatic.py
@@ -11,6 +11,9 @@ import filecmp
 import shutil
 import json
 import random
+import io
+import subprocess
+import tarfile
 
 import jinja2
 import jinja2.sandbox
@@ -40,11 +43,98 @@ global_filters = {
 }
 
 
+
+# Wrap operations on a generic directory
+class SourceWrapper(object):
+	def __init__(self, root):
+		self.root = root
+
+	def isdir(self, d):
+		return os.path.isdir(os.path.join(self.root, d))
+
+	def walkfiles(self, d):
+		relroot = os.path.join(self.root, d)
+		for dn, subdirs, filenames in os.walk(relroot):
+			relpath = os.path.relpath(dn, self.root)
+			for fn in filenames:
+				yield (relpath, fn)
+
+	def listfiles(self, d):
+		return os.listdir(os.path.join(self.root, d))
+
+	def copy_if_changed(self, relsource, fulldest):
+		fullsrc = os.path.join(self.root, relsource)
+		if (not os.path.exists(fulldest)) or (not filecmp.cmp(fullsrc, fulldest)):
+			shutil.copy2(fullsrc, fulldest)
+
+	def readfile(self, src):
+		if os.path.isfile(os.path.join(self.root, src)):
+			with open(os.path.join(self.root, src)) as f:
+				return f.read()
+		else:
+			return None
+
+# Wrap operations on a tarfile
+class TarWrapper(object):
+	def __init__(self, tarstream):
+		self.tardata = io.BytesIO()
+		shutil.copyfileobj(tarstream, self.tardata)
+		self.tardata.seek(0)
+		self.tarfile = tarfile.open(fileobj=self.tardata)
+
+		self.tarstruct = {}
+		for m in self.tarfile.getmembers():
+			self.tarstruct[m.name] = m
+
+	def isdir(self, d):
+		return self.tarstruct.has_key(d) and self.tarstruct[d].isdir()
+
+	def isfile(self, f):
+		return self.tarstruct.has_key(f) and self.tarstruct[f].isfile()
+
+	def readfile(self, src):
+		if self.tarstruct.has_key(src) and self.tarstruct[src].isfile():
+			return self.tarfile.extractfile(src).read()
+		else:
+			return None
+
+	def walkfiles(self, d):
+		for k in self.tarstruct.keys():
+			if k.startswith(d + '/') and self.tarstruct[k].isfile():
+				yield (os.path.dirname(k), os.path.basename(k))
+
+	def listfiles(self, d):
+		for k in self.tarstruct.keys():
+			if os.path.dirname(k) == d:
+				yield os.path.basename(k)
+
+	def copy_if_changed(self, relsource, fulldest):
+		sourcedata = self.readfile(relsource)
+
+		if os.path.isfile(fulldest):
+			with open(fulldest, 'r') as f:
+				x = f.read()
+				if x == sourcedata:
+					return
+
+		with open(fulldest, 'w') as f:
+			f.write(sourcedata)
+
+
+class JinjaTarLoader(jinja2.BaseLoader):
+	def __init__(self, tarwrapper):
+		self.tarwrapper = tarwrapper
+
+	def get_source(self, environment, template):
+		t = os.path.join('templates/', template)
+		if self.tarwrapper.isfile(t):
+			return (self.tarwrapper.readfile(t).decode('utf8'), None, None)
+		raise jinja2.TemplateNotFound(template)
+
 # Optionally load a JSON context
-def load_context(jsonfile):
-	if os.path.isfile(jsonfile):
-		with open(jsonfile) as f:
-			return json.load(f)
+def load_context(jsondata):
+	if jsondata:
+		return json.loads(jsondata)
 	else:
 		return {}
 
@@ -88,6 +178,7 @@ if __name__ == "__main__":
 	parser = argparse.ArgumentParser(description='Deploy jinja based static site')
 	parser.add_argument('sourcepath', type=str, help='Source path')
 	parser.add_argument('destpath', type=str, help='Destination path')
+	parser.add_argument('--branch', type=str, help='Deploy directly from branch')
 
 	args = parser.parse_args()
 
@@ -106,45 +197,52 @@ if __name__ == "__main__":
 		print "Destination directory does not exist!"
 		sys.exit(1)
 
+	if args.branch:
+		s = subprocess.Popen(['/usr/bin/git', 'archive', '--format=tar', args.branch],
+							 stdout=subprocess.PIPE,
+							 cwd=args.sourcepath)
+		source = TarWrapper(s.stdout)
+		s.stdout.close()
+	else:
+		source = SourceWrapper(args.sourcepath)
+
 	for d in ('templates', 'templates/pages', 'static'):
-		if not os.path.isdir(os.path.join(args.sourcepath, d)):
+		if not source.isdir(d):
 			print "'{0}' subdirectory does not exist in source!".format(d)
 			sys.exit(1)
 
-	staticroot = os.path.join(args.sourcepath, 'static/')
 	staticdest = os.path.join(args.destpath, 'static/')
 
 	# Set up jinja environment
-	env = jinja2.sandbox.SandboxedEnvironment(loader=jinja2.FileSystemLoader([os.path.join(args.sourcepath, 'templates/'),]))
+	if args.branch:
+		env = jinja2.sandbox.SandboxedEnvironment(loader=JinjaTarLoader(source))
+	else:
+		env = jinja2.sandbox.SandboxedEnvironment(loader=jinja2.FileSystemLoader([os.path.join(args.sourcepath, 'templates/'),]))
 	env.filters.update(global_filters)
 
 	# If there is a context json, load it as well
-	context = load_context(os.path.join(args.sourcepath, 'templates', 'context.json'))
+	context = load_context(source.readfile('templates/context.json'))
 
 	# Fetch the current git revision if this is coming out of a git repository
 	context['githash'] = find_git_revision(args.sourcepath)
 
 	# Load a context that can override everything, including static hashes
-	context.update(load_context(os.path.join(args.sourcepath, 'templates', 'context.override.json')))
+	context.update(load_context(source.readfile('templates/context.override.json')))
 
 
 	knownfiles = []
 	# We could use copytree(), but we need to know which files are there so we can
 	# remove old files, so we might as well do the full processing this way.
-	for dn, subdirs, filenames in os.walk(staticroot):
-		relpath = os.path.relpath(dn, staticroot)
-		if not os.path.isdir(os.path.join(staticdest, relpath)):
-			os.makedirs(os.path.join(staticdest, relpath))
+	for relpath, relname in source.walkfiles('static'):
+		if not os.path.isdir(os.path.join(args.destpath, relpath)):
+			os.makedirs(os.path.join(args.destpath, relpath))
 
-		for fn in filenames:
-			fullsrc = os.path.join(staticroot, relpath, fn)
-			fulldest = os.path.join(staticdest, relpath, fn)
-			if (not os.path.exists(fulldest)) or (not filecmp.cmp(fullsrc, fulldest)):
-				shutil.copy2(fullsrc, fulldest)
-			knownfiles.append(os.path.join('static', relpath, fn))
-
-	pagesroot = os.path.join(args.sourcepath, 'templates/pages')
-	for fn in os.listdir(pagesroot):
+		relsource = os.path.join(relpath, relname)
+		source.copy_if_changed(relsource, os.path.join(args.destpath, relsource))
+
+		knownfiles.append(relsource)
+
+	for fn in source.listfiles('templates/pages'):
 		# We don't use subdirectories yet, so don't bother even looking at that
 		if os.path.splitext(fn)[1] != '.html':
 			continue