Hacky/Preliminary inlining implementation.jit-01

6 files changed, 391 insertions, 4 deletions

diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index ab5862b472..0d47734a6a 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -812,6 +812,10 @@ ifndef COMPILE.c
 COMPILE.c = $(CC) $(CFLAGS) $(CPPFLAGS) -c
 endif
 
+ifndef COMPILE.bc
+COMPILE.bc = $(CC) $(filter-out -g3 -g -ggdb3 -fno-strict-aliasing, $(CFLAGS)) -fstrict-aliasing $(CPPFLAGS) -emit-llvm -c
+endif
+
 DEPDIR = .deps
 
 ifeq ($(GCC), yes)
@@ -821,6 +825,10 @@ ifeq ($(GCC), yes)
 	@if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi
 	$(COMPILE.c) -o $@ $< -MMD -MP -MF $(DEPDIR)/$(*F).Po
 
+%.bc : %.c %.o
+	@if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi
+	$(COMPILE.bc) -o $@ $<
+
 endif # GCC
 
 # Include all the dependency files generated for the current
diff --git a/src/backend/Makefile b/src/backend/Makefile
index c82ad75bda..1f1945b5f3 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -64,6 +64,12 @@ ifneq ($(PORTNAME), aix)
 postgres: $(OBJS)
 	$(CC) $(CFLAGS) $(LDFLAGS) $(LDFLAGS_EX) $(export_dynamic) $(call expand_subsys,$^) $(LIBS) -o $@
 
+
+ifeq ($(findstring clang,$(CC)), clang)
+parsed: $(OBJS)
+	$(MAKE $(patsubst %.o,%.bc,$(call expand_subsys,$(OBJS))))
+endif
+
 endif
 endif
 endif
diff --git a/src/backend/executor/execExprCompile.c b/src/backend/executor/execExprCompile.c
index d0b943530c..73412d0e1b 100644
--- a/src/backend/executor/execExprCompile.c
+++ b/src/backend/executor/execExprCompile.c
@@ -213,9 +213,21 @@ BuildFunctionCall(LLVMJitContext *context, LLVMBuilderRef builder,
 	}
 	else if (builtin)
 	{
+		LLVMModuleRef inline_mod;
+
 		LLVMAddFunction(mod, builtin->funcName, TypePGFunction);
 		v_fn_addr = LLVMGetNamedFunction(mod, builtin->funcName);
 		Assert(v_fn_addr);
+
+		inline_mod = llvm_module_for_function(builtin->funcName);
+		if (inline_mod)
+		{
+			context->inline_modules =
+				list_append_unique_ptr(context->inline_modules,
+									   inline_mod);
+		}
+
+		forceinline = true;
 	}
 	else
 	{
@@ -263,6 +275,20 @@ BuildFunctionCall(LLVMJitContext *context, LLVMBuilderRef builder,
 		LLVMValueRef v_lifetime = get_LifetimeEnd(mod);
 		LLVMValueRef params[2];
 
+		params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(fcinfo->arg), false);
+		params[1] = LLVMBuildBitCast(
+			builder, LLVMConstInt(TypeSizeT, (intptr_t) fcinfo->arg, false),
+			LLVMPointerType(LLVMInt8Type(), 0),
+			"");
+		LLVMBuildCall(builder, v_lifetime, params, lengthof(params), "");
+
+		params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(fcinfo->argnull), false);
+		params[1] = LLVMBuildBitCast(
+			builder, LLVMConstInt(TypeSizeT, (intptr_t) fcinfo->argnull, false),
+			LLVMPointerType(LLVMInt8Type(), 0),
+			"");
+		LLVMBuildCall(builder, v_lifetime, params, lengthof(params), "");
+
 		params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(FunctionCallInfoData), false);
 		params[1] = LLVMBuildBitCast(
 			builder, v_fcinfo,
@@ -314,6 +340,9 @@ ExecRunCompiledExpr(ExprState *state, ExprContext *econtext, bool *isNull)
 	return func(state, econtext, isNull);
 }
 
+static void
+emit_lifetime_end(ExprState *state, LLVMModuleRef mod, LLVMBuilderRef builder);
+
 bool
 ExecReadyCompiledExpr(ExprState *state, PlanState *parent)
 {
@@ -519,6 +548,7 @@ ExecReadyCompiledExpr(ExprState *state, PlanState *parent)
 						LLVMBuildCall(builder, v_lifetime, params, lengthof(params), "");
 					}
 
+					emit_lifetime_end(state, mod, builder);
 
 					LLVMBuildRet(builder, v_tmpvalue);
 					break;
@@ -2793,4 +2823,85 @@ ExecReadyCompiledExpr(ExprState *state, PlanState *parent)
 	return true;
 }
 
+
+static void
+emit_lifetime_end(ExprState *state, LLVMModuleRef mod, LLVMBuilderRef builder)
+{
+	ExprEvalStep   *op;
+	int i = 0;
+	int argno = 0;
+	LLVMValueRef v_lifetime = get_LifetimeEnd(mod);
+
+
+	/*
+	 * Add lifetime-end annotation, signalling that writes to memory don't
+	 * have to be retained (important for inlining potential).
+	 */
+
+	for (i = 0; i < state->steps_len; i++)
+	{
+		FunctionCallInfo fcinfo = NULL;
+		LLVMValueRef v_ptr;
+		LLVMValueRef params[2];
+
+		op = &state->steps[i];
+
+		switch ((ExprEvalOp) op->opcode)
+		{
+			case EEOP_FUNCEXPR:
+			case EEOP_FUNCEXPR_STRICT:
+			case EEOP_NULLIF:
+			case EEOP_DISTINCT:
+				fcinfo = op->d.func.fcinfo_data;
+
+				for (argno = 0; argno < op->d.func.nargs; argno++)
+				{
+					params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(Datum), false);
+					params[1] = LLVMBuildIntToPtr(
+						builder, LLVMConstInt(TypeSizeT, (intptr_t) &fcinfo->arg[argno], false),
+						LLVMPointerType(LLVMInt8Type(), 0),
+						"");
+					LLVMBuildCall(builder, v_lifetime, params, lengthof(params), "");
+
+					params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(bool), false);
+					params[1] = LLVMBuildIntToPtr(
+						builder, LLVMConstInt(TypeSizeT, (intptr_t) &fcinfo->argnull[argno], false),
+						LLVMPointerType(LLVMInt8Type(), 0),
+						"");
+					LLVMBuildCall(builder, v_lifetime, params, lengthof(params), "");
+				}
+				params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(FunctionCallInfoData), false);
+				params[1] = LLVMBuildIntToPtr(
+					builder, LLVMConstInt(TypeSizeT, (intptr_t) fcinfo, false),
+					LLVMPointerType(LLVMInt8Type(), 0),
+					"");
+				LLVMBuildCall(builder, v_lifetime, params, lengthof(params), "");
+
+				break;
+			case EEOP_ROWCOMPARE_STEP:
+				fcinfo = op->d.rowcompare_step.fcinfo_data;;
+				break;
+			case EEOP_BOOLTEST_IS_TRUE:
+			case EEOP_BOOLTEST_IS_NOT_FALSE:
+			case EEOP_BOOLTEST_IS_FALSE:
+			case EEOP_BOOLTEST_IS_NOT_TRUE:
+				if (op->d.boolexpr.anynull)
+				{
+					v_ptr = LLVMBuildIntToPtr(
+						builder,
+						LLVMConstInt(LLVMInt64Type(), (intptr_t) op->d.boolexpr.anynull, false),
+						LLVMPointerType(LLVMInt8Type(), 0),
+						"anynull");
+
+					params[0] = LLVMConstInt(LLVMInt64Type(), sizeof(bool), false);
+					params[1] = v_ptr;
+					LLVMBuildCall(builder, v_lifetime, params, lengthof(params), "");
+				}
+				break;
+		default:
+			break;
+		}
+
+	}
+}
 #endif
diff --git a/src/backend/lib/llvmjit.c b/src/backend/lib/llvmjit.c
index 57d0663410..86d43c3c07 100644
--- a/src/backend/lib/llvmjit.c
+++ b/src/backend/lib/llvmjit.c
@@ -9,6 +9,7 @@
 
 #include "utils/memutils.h"
 #include "utils/resowner_private.h"
+#include "utils/varlena.h"
 
 #ifdef USE_LLVM
 
@@ -32,6 +33,8 @@
 /* GUCs */
 bool jit_log_ir = 0;
 bool jit_dump_bitcode = 0;
+bool jit_perform_inlining = 0;
+char *jit_inline_directories = NULL;
 
 static bool llvm_initialized = false;
 static LLVMPassManagerBuilderRef llvm_pmb;
@@ -72,6 +75,8 @@ static LLVMOrcJITStackRef llvm_orc;
 static void llvm_shutdown(void);
 static void llvm_create_types(void);
 
+static void llvm_search_inline_directories(void);
+
 
 static void
 llvm_shutdown(void)
@@ -103,6 +108,8 @@ llvm_initialize(void)
 	LLVMLoadLibraryPermanently("");
 
 	llvm_triple = LLVMGetDefaultTargetTriple();
+	/* FIXME: overwrite with clang compatible one? */
+	llvm_triple = "x86_64-pc-linux-gnu";
 
 	if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
 	{
@@ -117,6 +124,7 @@ llvm_initialize(void)
 
 	llvm_pmb = LLVMPassManagerBuilderCreate();
 	LLVMPassManagerBuilderSetOptLevel(llvm_pmb, 3);
+	LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 0);
 
 	llvm_orc = LLVMOrcCreateInstance(llvm_targetmachine);
 
@@ -128,9 +136,188 @@ llvm_initialize(void)
 	llvm_create_types();
 
 	llvm_initialized = true;
+
+	llvm_search_inline_directories();
+
 	MemoryContextSwitchTo(oldcontext);
 }
 
+#include "common/string.h"
+#include "storage/fd.h"
+#include "miscadmin.h"
+
+static HTAB *InlineModuleHash = NULL;
+
+typedef struct InlineableFunction
+{
+	NameData fname;
+	const char *path;
+	LLVMModuleRef mod;
+} InlineableFunction;
+
+static void
+llvm_preload_bitcode(const char *filename)
+{
+	LLVMMemoryBufferRef buf;
+	char *msg;
+	LLVMValueRef func;
+	LLVMModuleRef mod = NULL;
+
+	mod = LLVMModuleCreateWithName("tmp");
+
+	if (LLVMCreateMemoryBufferWithContentsOfFile(
+			filename, &buf, &msg))
+	{
+		elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
+			 filename, msg);
+	}
+
+#if 1
+	if (LLVMParseBitcode2(buf, &mod))
+	{
+		elog(ERROR, "LLVMParseBitcode2 failed: %s", msg);
+	}
+#else
+	if (LLVMGetBitcodeModule2(buf, &mod))
+	{
+		elog(ERROR, "LLVMGetBitcodeModule2 failed: %s", msg);
+	}
+#endif
+
+	func = LLVMGetFirstFunction(mod);
+	while (func)
+	{
+		const char *funcname = LLVMGetValueName(func);
+
+		if (!LLVMIsDeclaration(func))
+		{
+			if (LLVMGetLinkage(func) == LLVMExternalLinkage)
+			{
+				InlineableFunction *fentry;
+				bool found;
+
+				fentry = (InlineableFunction *)
+					hash_search(InlineModuleHash,
+								(void *) funcname,
+								HASH_ENTER, &found);
+
+				if (found)
+				{
+					elog(LOG, "skiping loading func %s, already exists at %s, loading %s",
+						 funcname, fentry->path, filename);
+				}
+				else
+				{
+					fentry->path = pstrdup(filename);
+					fentry->mod = mod;
+				}
+
+				LLVMSetLinkage(func, LLVMAvailableExternallyLinkage);
+			}
+		}
+
+		func = LLVMGetNextFunction(func);
+	}
+}
+
+static void
+llvm_search_inline_directory(const char *path)
+{
+	DIR		   *dir;
+	struct dirent *de;
+
+	dir = AllocateDir(path);
+	if (dir == NULL)
+	{
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open directory \"%s\": %m", path)));
+		return;
+	}
+
+	while ((de = ReadDir(dir, path)) != NULL)
+	{
+		char		subpath[MAXPGPATH * 2];
+		struct stat fst;
+		int			sret;
+
+		CHECK_FOR_INTERRUPTS();
+
+		if (strcmp(de->d_name, ".") == 0 ||
+			strcmp(de->d_name, "..") == 0)
+			continue;
+
+		snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name);
+
+		sret = lstat(subpath, &fst);
+
+		if (sret < 0)
+		{
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not stat file \"%s\": %m", subpath)));
+			continue;
+		}
+
+		if (S_ISREG(fst.st_mode))
+		{
+			if (pg_str_endswith(subpath, ".bc"))
+			{
+				llvm_preload_bitcode(subpath);
+			}
+		}
+		else if (S_ISDIR(fst.st_mode))
+		{
+			llvm_search_inline_directory(subpath);
+		}
+	}
+
+	FreeDir(dir);				/* we ignore any error here */
+}
+
+static void
+llvm_search_inline_directories(void)
+{
+	List *elemlist;
+	ListCell *lc;
+	HASHCTL		ctl;
+
+	Assert(InlineModuleHash == NULL);
+	/* First time through: initialize the hash table */
+
+	MemSet(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(NameData);
+	ctl.entrysize = sizeof(InlineableFunction);
+	InlineModuleHash = hash_create("inlineable function cache", 64,
+								   &ctl, HASH_ELEM);
+
+	SplitDirectoriesString(pstrdup(jit_inline_directories), ';', &elemlist);
+
+	foreach(lc, elemlist)
+	{
+		char	   *curdir = (char *) lfirst(lc);
+
+		llvm_search_inline_directory(curdir);
+	}
+}
+
+LLVMModuleRef
+llvm_module_for_function(const char *funcname)
+{
+	InlineableFunction *fentry;
+	bool found;
+
+	fentry = (InlineableFunction *)
+		hash_search(InlineModuleHash,
+					(void *) funcname,
+					HASH_FIND, &found);
+
+	if (fentry)
+		return fentry->mod;
+	return NULL;
+}
+
+
 static void
 llvm_create_types(void)
 {
@@ -399,7 +586,11 @@ llvm_create_types(void)
 static uint64_t
 llvm_resolve_symbol(const char *name, void *ctx)
 {
-	return (uint64_t) LLVMSearchForAddressOfSymbol(name);
+	uint64_t addr = (uint64_t) LLVMSearchForAddressOfSymbol(name);
+
+	if (!addr)
+		elog(ERROR, "failed to resolve name %s", name);
+	return addr;
 }
 
 void *
@@ -412,9 +603,22 @@ llvm_get_function(LLVMJitContext *context, const char *funcname)
 	if (!context->compiled)
 	{
 		int handle;
-		LLVMSharedModuleRef smod = LLVMOrcMakeSharedModule(context->module);
+		LLVMSharedModuleRef smod;
 		MemoryContext oldcontext;
 
+		if (jit_perform_inlining)
+		{
+			ListCell *lc;
+
+			foreach(lc, context->inline_modules)
+			{
+				LLVMModuleRef inline_mod = lfirst(lc);
+
+				inline_mod = LLVMCloneModule(inline_mod);
+				LLVMLinkModules2Needed(context->module, inline_mod);
+			}
+		}
+
 		if (jit_log_ir)
 		{
 			LLVMDumpModule(context->module);
@@ -439,13 +643,26 @@ llvm_get_function(LLVMJitContext *context, const char *funcname)
 			llvm_mpm = LLVMCreatePassManager();
 
 			LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
-			LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb, llvm_mpm);
+			//LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb, llvm_mpm);
 			LLVMPassManagerBuilderPopulateLTOPassManager(llvm_pmb, llvm_mpm, true, true);
 
+			LLVMAddCFGSimplificationPass(llvm_fpm);
+			LLVMAddJumpThreadingPass(llvm_fpm);
+			LLVMAddTypeBasedAliasAnalysisPass(llvm_fpm);
+			LLVMAddDeadStoreEliminationPass(llvm_fpm);
+			LLVMAddConstantPropagationPass(llvm_fpm);
+			LLVMAddSCCPPass(llvm_fpm);
+
 			LLVMAddAnalysisPasses(llvm_targetmachine, llvm_mpm);
 			LLVMAddAnalysisPasses(llvm_targetmachine, llvm_fpm);
 
-			LLVMAddDeadStoreEliminationPass(llvm_fpm);
+			/* do function level optimization */
+			LLVMInitializeFunctionPassManager(llvm_fpm);
+			for (func = LLVMGetFirstFunction(context->module);
+				 func != NULL;
+				 func = LLVMGetNextFunction(func))
+				LLVMRunFunctionPassManager(llvm_fpm, func);
+			LLVMFinalizeFunctionPassManager(llvm_fpm);
 
 			/* do function level optimization */
 			LLVMInitializeFunctionPassManager(llvm_fpm);
@@ -457,11 +674,24 @@ llvm_get_function(LLVMJitContext *context, const char *funcname)
 
 			/* do module level optimization */
 			LLVMRunPassManager(llvm_mpm, context->module);
+			LLVMRunPassManager(llvm_mpm, context->module);
+			LLVMRunPassManager(llvm_mpm, context->module);
+			LLVMRunPassManager(llvm_mpm, context->module);
 
 			LLVMDisposePassManager(llvm_fpm);
 			LLVMDisposePassManager(llvm_mpm);
 		}
 
+		if (jit_dump_bitcode)
+		{
+			/* FIXME: invent module rather than function specific name */
+			char *filename = psprintf("%s.optimized.bc", funcname);
+			LLVMWriteBitcodeToFile(context->module, filename);
+			pfree(filename);
+		}
+
+		smod = LLVMOrcMakeSharedModule(context->module);
+
 		/* and emit the code */
 		{
 			handle =
@@ -480,6 +710,7 @@ llvm_get_function(LLVMJitContext *context, const char *funcname)
 
 		context->module = NULL;
 		context->compiled = true;
+		context->inline_modules = NIL;
 	}
 
 	/* search all emitted modules for function we're asked for */
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 4cc9f305a2..82359b1616 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -1043,6 +1043,17 @@ static struct config_bool ConfigureNamesBool[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"jit_perform_inlining", PGC_USERSET, DEVELOPER_OPTIONS,
+			gettext_noop("inline functions for JIT"),
+			NULL,
+			GUC_NOT_IN_SAMPLE
+		},
+		&jit_perform_inlining,
+		false,
+		NULL, NULL, NULL
+	},
+
 #endif
 
 	{
@@ -3700,6 +3711,19 @@ static struct config_string ConfigureNamesString[] =
 		check_wal_consistency_checking, assign_wal_consistency_checking, NULL
 	},
 
+#ifdef USE_LLVM
+	{
+		{"jit_inline_directories", PGC_BACKEND, DEVELOPER_OPTIONS,
+			gettext_noop("Sets the directories where inline contents for JIT are located."),
+			NULL,
+			GUC_SUPERUSER_ONLY
+		},
+		&jit_inline_directories,
+		"",
+		NULL, NULL, NULL
+	},
+#endif
+
 	/* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL
diff --git a/src/include/lib/llvmjit.h b/src/include/lib/llvmjit.h
index 47f9b6d64c..e2a8cccafc 100644
--- a/src/include/lib/llvmjit.h
+++ b/src/include/lib/llvmjit.h
@@ -27,12 +27,15 @@ typedef struct LLVMJitContext
 {
 	int counter;
 	LLVMModuleRef module;
+	List *inline_modules;
 	bool compiled;
 	List *handles;
 } LLVMJitContext;
 
 extern bool jit_log_ir;
 extern bool jit_dump_bitcode;
+extern bool jit_perform_inlining;
+extern char *jit_inline_directories;
 
 extern LLVMTargetMachineRef llvm_targetmachine;
 extern const char *llvm_triple;
@@ -74,6 +77,10 @@ extern void llvm_shutdown_orc_perf_support(LLVMOrcJITStackRef llvm_orc);
 
 extern LLVMValueRef slot_compile_deform(struct LLVMJitContext *context, TupleDesc desc, int natts);
 
+
+extern LLVMModuleRef llvm_module_for_function(const char *funcname);
+
+
 #else
 
 struct LLVMJitContext;