From c1156411ad0879a71956b64aa487babe7572685b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 24 Mar 2016 20:28:47 -0400 Subject: Move psql's psqlscan.l into src/fe_utils. This completes (at least for now) the project of getting rid of ad-hoc linkages among the src/bin/ subdirectories. Everything they share is now in src/fe_utils/ and is included from a static library at link time. A side benefit is that we can restore the FLEX_NO_BACKUP check for psqlscanslash.l. We might need to think of another way to do that check if we ever need to build two lexers with that property in the same source directory, but there's no foreseeable reason to need that. --- src/backend/parser/scan.l | 2 +- src/bin/pgbench/Makefile | 17 +- src/bin/pgbench/exprscan.l | 2 +- src/bin/pgbench/pgbench.h | 2 +- src/bin/psql/.gitignore | 1 - src/bin/psql/Makefile | 15 +- src/bin/psql/command.h | 2 +- src/bin/psql/mainloop.c | 1 + src/bin/psql/mainloop.h | 2 +- src/bin/psql/nls.mk | 4 +- src/bin/psql/prompt.h | 13 +- src/bin/psql/psqlscan.h | 62 -- src/bin/psql/psqlscan.l | 1424 ---------------------------------- src/bin/psql/psqlscan_int.h | 129 ---- src/bin/psql/psqlscanslash.h | 2 +- src/bin/psql/psqlscanslash.l | 4 +- src/fe_utils/.gitignore | 1 + src/fe_utils/Makefile | 27 +- src/fe_utils/psqlscan.l | 1428 +++++++++++++++++++++++++++++++++++ src/include/fe_utils/psqlscan.h | 85 +++ src/include/fe_utils/psqlscan_int.h | 144 ++++ src/tools/msvc/Mkvcbuild.pm | 17 +- src/tools/msvc/clean.bat | 2 +- 23 files changed, 1708 insertions(+), 1678 deletions(-) delete mode 100644 src/bin/psql/psqlscan.h delete mode 100644 src/bin/psql/psqlscan.l delete mode 100644 src/bin/psql/psqlscan_int.h create mode 100644 src/fe_utils/.gitignore create mode 100644 src/fe_utils/psqlscan.l create mode 100644 src/include/fe_utils/psqlscan.h create mode 100644 src/include/fe_utils/psqlscan_int.h (limited to 'src') diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l index 8f2d075d438..4d905e6b94f 100644 --- a/src/backend/parser/scan.l +++ b/src/backend/parser/scan.l @@ -6,7 +6,7 @@ * * NOTE NOTE NOTE: * - * The rules in this file must be kept in sync with psql's psqlscan.l! + * The rules in this file must be kept in sync with src/fe_utils/psqlscan.l! * * The rules are designed so that the scanner never has to backtrack, * in the sense that there is always a rule that can match the input diff --git a/src/bin/pgbench/Makefile b/src/bin/pgbench/Makefile index e5d22c2e7c2..5e608b654b9 100644 --- a/src/bin/pgbench/Makefile +++ b/src/bin/pgbench/Makefile @@ -7,10 +7,10 @@ subdir = src/bin/pgbench top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -OBJS = pgbench.o exprparse.o psqlscan.o $(WIN32RES) +OBJS = pgbench.o exprparse.o $(WIN32RES) -override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) \ - -I$(top_srcdir)/src/bin/psql $(CPPFLAGS) +override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS) +LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils ifneq ($(PORTNAME), win32) override CFLAGS += $(PTHREAD_CFLAGS) @@ -19,21 +19,12 @@ endif all: pgbench -pgbench: $(OBJS) | submake-libpq submake-libpgport +pgbench: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils $(CC) $(CFLAGS) $^ $(libpq_pgport) $(PTHREAD_LIBS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) # exprscan is compiled as part of exprparse exprparse.o: exprscan.c -# we import psqlscan.o as-is from psql -submake-psqlscan: - $(MAKE) -C $(top_builddir)/src/bin/psql psqlscan.o - -psqlscan.o: | submake-psqlscan - rm -f $@ && $(LN_S) $(top_builddir)/src/bin/psql/psqlscan.o . - -.PHONY: submake-psqlscan - distprep: exprparse.c exprscan.c install: all installdirs diff --git a/src/bin/pgbench/exprscan.l b/src/bin/pgbench/exprscan.l index 825dacc0af3..7120836f840 100644 --- a/src/bin/pgbench/exprscan.l +++ b/src/bin/pgbench/exprscan.l @@ -23,7 +23,7 @@ *------------------------------------------------------------------------- */ -#include "psqlscan_int.h" +#include "fe_utils/psqlscan_int.h" /* context information for reporting errors in expressions */ static const char *expr_source = NULL; diff --git a/src/bin/pgbench/pgbench.h b/src/bin/pgbench/pgbench.h index 101b0bdbf3b..46350aace10 100644 --- a/src/bin/pgbench/pgbench.h +++ b/src/bin/pgbench/pgbench.h @@ -11,7 +11,7 @@ #ifndef PGBENCH_H #define PGBENCH_H -#include "psqlscan.h" +#include "fe_utils/psqlscan.h" /* * This file is included outside exprscan.l, in places where we can't see diff --git a/src/bin/psql/.gitignore b/src/bin/psql/.gitignore index 9de50c0a287..c2862b12d62 100644 --- a/src/bin/psql/.gitignore +++ b/src/bin/psql/.gitignore @@ -1,4 +1,3 @@ -/psqlscan.c /psqlscanslash.c /sql_help.h /sql_help.c diff --git a/src/bin/psql/Makefile b/src/bin/psql/Makefile index 4e68e3ebdca..6220d0d620f 100644 --- a/src/bin/psql/Makefile +++ b/src/bin/psql/Makefile @@ -24,7 +24,7 @@ LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils OBJS= command.o common.o help.o input.o stringutils.o mainloop.o copy.o \ startup.o prompt.o variables.o large_obj.o describe.o \ tab-complete.o \ - sql_help.o psqlscan.o psqlscanslash.o \ + sql_help.o psqlscanslash.o \ $(WIN32RES) @@ -39,20 +39,15 @@ sql_help.c: sql_help.h ; sql_help.h: create_help.pl $(wildcard $(REFDOCDIR)/*.sgml) $(PERL) $< $(REFDOCDIR) $* -psqlscan.c: FLEXFLAGS = -Cfe -p -p -psqlscan.c: FLEX_NO_BACKUP=yes - psqlscanslash.c: FLEXFLAGS = -Cfe -p -p -# Ideally we'd check this, but parallel make causes problems: -# psqlscanslash.c: FLEX_NO_BACKUP=yes +psqlscanslash.c: FLEX_NO_BACKUP=yes -# Latest flex causes warnings in these files. +# Latest flex causes warnings in this file. ifeq ($(GCC),yes) -psqlscan.o: CFLAGS += -Wno-error psqlscanslash.o: CFLAGS += -Wno-error endif -distprep: sql_help.h psqlscan.c psqlscanslash.c +distprep: sql_help.h psqlscanslash.c install: all installdirs $(INSTALL_PROGRAM) psql$(X) '$(DESTDIR)$(bindir)/psql$(X)' @@ -70,4 +65,4 @@ clean distclean: # files removed here are supposed to be in the distribution tarball, # so do not clean them in the clean/distclean rules maintainer-clean: distclean - rm -f sql_help.h sql_help.c psqlscan.c psqlscanslash.c + rm -f sql_help.h sql_help.c psqlscanslash.c diff --git a/src/bin/psql/command.h b/src/bin/psql/command.h index 4f0140fd563..8d37a3a676c 100644 --- a/src/bin/psql/command.h +++ b/src/bin/psql/command.h @@ -9,7 +9,7 @@ #define COMMAND_H #include "fe_utils/print.h" -#include "psqlscan.h" +#include "fe_utils/psqlscan.h" typedef enum _backslashResult diff --git a/src/bin/psql/mainloop.c b/src/bin/psql/mainloop.c index 1eb5a2ec537..37dfa4d0e35 100644 --- a/src/bin/psql/mainloop.c +++ b/src/bin/psql/mainloop.c @@ -11,6 +11,7 @@ #include "command.h" #include "common.h" #include "input.h" +#include "prompt.h" #include "settings.h" #include "mb/pg_wchar.h" diff --git a/src/bin/psql/mainloop.h b/src/bin/psql/mainloop.h index 5ee8dc7f63f..fd9723a25ce 100644 --- a/src/bin/psql/mainloop.h +++ b/src/bin/psql/mainloop.h @@ -8,7 +8,7 @@ #ifndef MAINLOOP_H #define MAINLOOP_H -#include "psqlscan.h" +#include "fe_utils/psqlscan.h" extern const PsqlScanCallbacks psqlscan_callbacks; diff --git a/src/bin/psql/nls.mk b/src/bin/psql/nls.mk index b9a7992a4fa..917ac0fc742 100644 --- a/src/bin/psql/nls.mk +++ b/src/bin/psql/nls.mk @@ -2,10 +2,10 @@ CATALOG_NAME = psql AVAIL_LANGUAGES = cs de es fr it ja pl pt_BR ru zh_CN zh_TW GETTEXT_FILES = command.c common.c copy.c help.c input.c large_obj.c \ - mainloop.c psqlscan.c psqlscanslash.c startup.c \ + mainloop.c psqlscanslash.c startup.c \ describe.c sql_help.h sql_help.c \ tab-complete.c variables.c \ - ../../fe_utils/print.c \ + ../../fe_utils/print.c ../../fe_utils/psqlscan.c \ ../../common/exec.c ../../common/fe_memutils.c ../../common/username.c \ ../../common/wait_error.c GETTEXT_TRIGGERS = N_ psql_error simple_prompt diff --git a/src/bin/psql/prompt.h b/src/bin/psql/prompt.h index 1f970a64f7e..d7e76dc181e 100644 --- a/src/bin/psql/prompt.h +++ b/src/bin/psql/prompt.h @@ -8,17 +8,8 @@ #ifndef PROMPT_H #define PROMPT_H -typedef enum _promptStatus -{ - PROMPT_READY, - PROMPT_CONTINUE, - PROMPT_COMMENT, - PROMPT_SINGLEQUOTE, - PROMPT_DOUBLEQUOTE, - PROMPT_DOLLARQUOTE, - PROMPT_PAREN, - PROMPT_COPY -} promptStatus_t; +/* enum promptStatus_t is now defined by psqlscan.h */ +#include "fe_utils/psqlscan.h" char *get_prompt(promptStatus_t status); diff --git a/src/bin/psql/psqlscan.h b/src/bin/psql/psqlscan.h deleted file mode 100644 index 4ff321866fc..00000000000 --- a/src/bin/psql/psqlscan.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * psql - the PostgreSQL interactive terminal - * - * Copyright (c) 2000-2016, PostgreSQL Global Development Group - * - * src/bin/psql/psqlscan.h - */ -#ifndef PSQLSCAN_H -#define PSQLSCAN_H - -#include "pqexpbuffer.h" - -#include "prompt.h" - - -/* Abstract type for lexer's internal state */ -typedef struct PsqlScanStateData *PsqlScanState; - -/* Termination states for psql_scan() */ -typedef enum -{ - PSCAN_SEMICOLON, /* found command-ending semicolon */ - PSCAN_BACKSLASH, /* found backslash command */ - PSCAN_INCOMPLETE, /* end of line, SQL statement incomplete */ - PSCAN_EOL /* end of line, SQL possibly complete */ -} PsqlScanResult; - -/* Callback functions to be used by the lexer */ -typedef struct PsqlScanCallbacks -{ - /* Fetch value of a variable, as a pfree'able string; NULL if unknown */ - /* This pointer can be NULL if no variable substitution is wanted */ - char *(*get_variable) (const char *varname, bool escape, bool as_ident); - /* Print an error message someplace appropriate */ - /* (very old gcc versions don't support attributes on function pointers) */ -#if defined(__GNUC__) && __GNUC__ < 4 - void (*write_error) (const char *fmt,...); -#else - void (*write_error) (const char *fmt,...) pg_attribute_printf(1, 2); -#endif -} PsqlScanCallbacks; - - -extern PsqlScanState psql_scan_create(const PsqlScanCallbacks *callbacks); -extern void psql_scan_destroy(PsqlScanState state); - -extern void psql_scan_setup(PsqlScanState state, - const char *line, int line_len, - int encoding, bool std_strings); -extern void psql_scan_finish(PsqlScanState state); - -extern PsqlScanResult psql_scan(PsqlScanState state, - PQExpBuffer query_buf, - promptStatus_t *prompt); - -extern void psql_scan_reset(PsqlScanState state); - -extern void psql_scan_reselect_sql_lexer(PsqlScanState state); - -extern bool psql_scan_in_quote(PsqlScanState state); - -#endif /* PSQLSCAN_H */ diff --git a/src/bin/psql/psqlscan.l b/src/bin/psql/psqlscan.l deleted file mode 100644 index 93c73559602..00000000000 --- a/src/bin/psql/psqlscan.l +++ /dev/null @@ -1,1424 +0,0 @@ -%top{ -/*------------------------------------------------------------------------- - * - * psqlscan.l - * lexical scanner for psql (and other frontend programs) - * - * This code is mainly needed to determine where the end of a SQL statement - * is: we are looking for semicolons that are not within quotes, comments, - * or parentheses. The most reliable way to handle this is to borrow the - * backend's flex lexer rules, lock, stock, and barrel. The rules below - * are (except for a few) the same as the backend's, but their actions are - * just ECHO whereas the backend's actions generally do other things. - * - * XXX The rules in this file must be kept in sync with the backend lexer!!! - * - * XXX Avoid creating backtracking cases --- see the backend lexer for info. - * - * See psqlscan_int.h for additional commentary. - * - * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * IDENTIFICATION - * src/bin/psql/psqlscan.l - * - *------------------------------------------------------------------------- - */ -#include "postgres_fe.h" - -#include "psqlscan.h" - -#include "libpq-fe.h" -} - -%{ -#include "psqlscan_int.h" - -/* - * We must have a typedef YYSTYPE for yylex's first argument, but this lexer - * doesn't presently make use of that argument, so just declare it as int. - */ -typedef int YYSTYPE; - -/* - * Set the type of yyextra; we use it as a pointer back to the containing - * PsqlScanState. - */ -#define YY_EXTRA_TYPE PsqlScanState - - -/* Return values from yylex() */ -#define LEXRES_EOL 0 /* end of input */ -#define LEXRES_SEMI 1 /* command-terminating semicolon found */ -#define LEXRES_BACKSLASH 2 /* backslash command start */ - - -static bool var_is_current_source(PsqlScanState state, const char *varname); - -#define ECHO psqlscan_emit(cur_state, yytext, yyleng) - -/* - * Work around a bug in flex 2.5.35: it emits a couple of functions that - * it forgets to emit declarations for. Since we use -Wmissing-prototypes, - * this would cause warnings. Providing our own declarations should be - * harmless even when the bug gets fixed. - */ -extern int psql_yyget_column(yyscan_t yyscanner); -extern void psql_yyset_column(int column_no, yyscan_t yyscanner); - -%} - -%option reentrant -%option bison-bridge -%option 8bit -%option never-interactive -%option nodefault -%option noinput -%option nounput -%option noyywrap -%option warn -%option prefix="psql_yy" - -/* - * All of the following definitions and rules should exactly match - * src/backend/parser/scan.l so far as the flex patterns are concerned. - * The rule bodies are just ECHO as opposed to what the backend does, - * however. (But be sure to duplicate code that affects the lexing process, - * such as BEGIN() and yyless().) Also, psqlscan uses a single <> rule - * whereas scan.l has a separate one for each exclusive state. - */ - -/* - * OK, here is a short description of lex/flex rules behavior. - * The longest pattern which matches an input string is always chosen. - * For equal-length patterns, the first occurring in the rules list is chosen. - * INITIAL is the starting state, to which all non-conditional rules apply. - * Exclusive states change parsing rules while the state is active. When in - * an exclusive state, only those rules defined for that state apply. - * - * We use exclusive states for quoted strings, extended comments, - * and to eliminate parsing troubles for numeric strings. - * Exclusive states: - * bit string literal - * extended C-style comments - * delimited identifiers (double-quoted identifiers) - * hexadecimal numeric string - * standard quoted strings - * extended quoted strings (support backslash escape sequences) - * $foo$ quoted strings - * quoted identifier with Unicode escapes - * end of a quoted identifier with Unicode escapes, UESCAPE can follow - * quoted string with Unicode escapes - * end of a quoted string with Unicode escapes, UESCAPE can follow - * - * Note: we intentionally don't mimic the backend's state; we have - * no need to distinguish it from state, and no good way to get out - * of it in error cases. The backend just throws yyerror() in those - * cases, but that's not an option here. - */ - -%x xb -%x xc -%x xd -%x xh -%x xe -%x xq -%x xdolq -%x xui -%x xuiend -%x xus -%x xusend - -/* - * In order to make the world safe for Windows and Mac clients as well as - * Unix ones, we accept either \n or \r as a newline. A DOS-style \r\n - * sequence will be seen as two successive newlines, but that doesn't cause - * any problems. Comments that start with -- and extend to the next - * newline are treated as equivalent to a single whitespace character. - * - * NOTE a fine point: if there is no newline following --, we will absorb - * everything to the end of the input as a comment. This is correct. Older - * versions of Postgres failed to recognize -- as a comment if the input - * did not end with a newline. - * - * XXX perhaps \f (formfeed) should be treated as a newline as well? - * - * XXX if you change the set of whitespace characters, fix scanner_isspace() - * to agree, and see also the plpgsql lexer. - */ - -space [ \t\n\r\f] -horiz_space [ \t\f] -newline [\n\r] -non_newline [^\n\r] - -comment ("--"{non_newline}*) - -whitespace ({space}+|{comment}) - -/* - * SQL requires at least one newline in the whitespace separating - * string literals that are to be concatenated. Silly, but who are we - * to argue? Note that {whitespace_with_newline} should not have * after - * it, whereas {whitespace} should generally have a * after it... - */ - -special_whitespace ({space}+|{comment}{newline}) -horiz_whitespace ({horiz_space}|{comment}) -whitespace_with_newline ({horiz_whitespace}*{newline}{special_whitespace}*) - -/* - * To ensure that {quotecontinue} can be scanned without having to back up - * if the full pattern isn't matched, we include trailing whitespace in - * {quotestop}. This matches all cases where {quotecontinue} fails to match, - * except for {quote} followed by whitespace and just one "-" (not two, - * which would start a {comment}). To cover that we have {quotefail}. - * The actions for {quotestop} and {quotefail} must throw back characters - * beyond the quote proper. - */ -quote ' -quotestop {quote}{whitespace}* -quotecontinue {quote}{whitespace_with_newline}{quote} -quotefail {quote}{whitespace}*"-" - -/* Bit string - * It is tempting to scan the string for only those characters - * which are allowed. However, this leads to silently swallowed - * characters if illegal characters are included in the string. - * For example, if xbinside is [01] then B'ABCD' is interpreted - * as a zero-length string, and the ABCD' is lost! - * Better to pass the string forward and let the input routines - * validate the contents. - */ -xbstart [bB]{quote} -xbinside [^']* - -/* Hexadecimal number */ -xhstart [xX]{quote} -xhinside [^']* - -/* National character */ -xnstart [nN]{quote} - -/* Quoted string that allows backslash escapes */ -xestart [eE]{quote} -xeinside [^\\']+ -xeescape [\\][^0-7] -xeoctesc [\\][0-7]{1,3} -xehexesc [\\]x[0-9A-Fa-f]{1,2} -xeunicode [\\](u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8}) -xeunicodefail [\\](u[0-9A-Fa-f]{0,3}|U[0-9A-Fa-f]{0,7}) - -/* Extended quote - * xqdouble implements embedded quote, '''' - */ -xqstart {quote} -xqdouble {quote}{quote} -xqinside [^']+ - -/* $foo$ style quotes ("dollar quoting") - * The quoted string starts with $foo$ where "foo" is an optional string - * in the form of an identifier, except that it may not contain "$", - * and extends to the first occurrence of an identical string. - * There is *no* processing of the quoted text. - * - * {dolqfailed} is an error rule to avoid scanner backup when {dolqdelim} - * fails to match its trailing "$". - */ -dolq_start [A-Za-z\200-\377_] -dolq_cont [A-Za-z\200-\377_0-9] -dolqdelim \$({dolq_start}{dolq_cont}*)?\$ -dolqfailed \${dolq_start}{dolq_cont}* -dolqinside [^$]+ - -/* Double quote - * Allows embedded spaces and other special characters into identifiers. - */ -dquote \" -xdstart {dquote} -xdstop {dquote} -xddouble {dquote}{dquote} -xdinside [^"]+ - -/* Unicode escapes */ -uescape [uU][eE][sS][cC][aA][pP][eE]{whitespace}*{quote}[^']{quote} -/* error rule to avoid backup */ -uescapefail [uU][eE][sS][cC][aA][pP][eE]{whitespace}*"-"|[uU][eE][sS][cC][aA][pP][eE]{whitespace}*{quote}[^']|[uU][eE][sS][cC][aA][pP][eE]{whitespace}*{quote}|[uU][eE][sS][cC][aA][pP][eE]{whitespace}*|[uU][eE][sS][cC][aA][pP]|[uU][eE][sS][cC][aA]|[uU][eE][sS][cC]|[uU][eE][sS]|[uU][eE]|[uU] - -/* Quoted identifier with Unicode escapes */ -xuistart [uU]&{dquote} - -/* Quoted string with Unicode escapes */ -xusstart [uU]&{quote} - -/* Optional UESCAPE after a quoted string or identifier with Unicode escapes. */ -xustop1 {uescapefail}? -xustop2 {uescape} - -/* error rule to avoid backup */ -xufailed [uU]& - - -/* C-style comments - * - * The "extended comment" syntax closely resembles allowable operator syntax. - * The tricky part here is to get lex to recognize a string starting with - * slash-star as a comment, when interpreting it as an operator would produce - * a longer match --- remember lex will prefer a longer match! Also, if we - * have something like plus-slash-star, lex will think this is a 3-character - * operator whereas we want to see it as a + operator and a comment start. - * The solution is two-fold: - * 1. append {op_chars}* to xcstart so that it matches as much text as - * {operator} would. Then the tie-breaker (first matching rule of same - * length) ensures xcstart wins. We put back the extra stuff with yyless() - * in case it contains a star-slash that should terminate the comment. - * 2. In the operator rule, check for slash-star within the operator, and - * if found throw it back with yyless(). This handles the plus-slash-star - * problem. - * Dash-dash comments have similar interactions with the operator rule. - */ -xcstart \/\*{op_chars}* -xcstop \*+\/ -xcinside [^*/]+ - -digit [0-9] -ident_start [A-Za-z\200-\377_] -ident_cont [A-Za-z\200-\377_0-9\$] - -identifier {ident_start}{ident_cont}* - -/* Assorted special-case operators and operator-like tokens */ -typecast "::" -dot_dot \.\. -colon_equals ":=" -equals_greater "=>" -less_equals "<=" -greater_equals ">=" -less_greater "<>" -not_equals "!=" - -/* - * "self" is the set of chars that should be returned as single-character - * tokens. "op_chars" is the set of chars that can make up "Op" tokens, - * which can be one or more characters long (but if a single-char token - * appears in the "self" set, it is not to be returned as an Op). Note - * that the sets overlap, but each has some chars that are not in the other. - * - * If you change either set, adjust the character lists appearing in the - * rule for "operator"! - */ -self [,()\[\].;\:\+\-\*\/\%\^\<\>\=] -op_chars [\~\!\@\#\^\&\|\`\?\+\-\*\/\%\<\>\=] -operator {op_chars}+ - -/* we no longer allow unary minus in numbers. - * instead we pass it separately to parser. there it gets - * coerced via doNegate() -- Leon aug 20 1999 - * - * {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot, 10. - * - * {realfail1} and {realfail2} are added to prevent the need for scanner - * backup when the {real} rule fails to match completely. - */ - -integer {digit}+ -decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*)) -decimalfail {digit}+\.\. -real ({integer}|{decimal})[Ee][-+]?{digit}+ -realfail1 ({integer}|{decimal})[Ee] -realfail2 ({integer}|{decimal})[Ee][-+] - -param \${integer} - -/* psql-specific: characters allowed in variable names */ -variable_char [A-Za-z\200-\377_0-9] - -other . - -/* - * Dollar quoted strings are totally opaque, and no escaping is done on them. - * Other quoted strings must allow some special characters such as single-quote - * and newline. - * Embedded single-quotes are implemented both in the SQL standard - * style of two adjacent single quotes "''" and in the Postgres/Java style - * of escaped-quote "\'". - * Other embedded escaped characters are matched explicitly and the leading - * backslash is dropped from the string. - * Note that xcstart must appear before operator, as explained above! - * Also whitespace (comment) must appear before operator. - */ - -%% - -%{ - /* Declare some local variables inside yylex(), for convenience */ - PsqlScanState cur_state = yyextra; - PQExpBuffer output_buf = cur_state->output_buf; - - /* - * Force flex into the state indicated by start_state. This has a - * couple of purposes: it lets some of the functions below set a new - * starting state without ugly direct access to flex variables, and it - * allows us to transition from one flex lexer to another so that we - * can lex different parts of the source string using separate lexers. - */ - BEGIN(cur_state->start_state); -%} - -{whitespace} { - /* - * Note that the whitespace rule includes both true - * whitespace and single-line ("--" style) comments. - * We suppress whitespace at the start of the query - * buffer. We also suppress all single-line comments, - * which is pretty dubious but is the historical - * behavior. - */ - if (!(output_buf->len == 0 || yytext[0] == '-')) - ECHO; - } - -{xcstart} { - cur_state->xcdepth = 0; - BEGIN(xc); - /* Put back any characters past slash-star; see above */ - yyless(2); - ECHO; - } - -{xcstart} { - cur_state->xcdepth++; - /* Put back any characters past slash-star; see above */ - yyless(2); - ECHO; - } - -{xcstop} { - if (cur_state->xcdepth <= 0) - BEGIN(INITIAL); - else - cur_state->xcdepth--; - ECHO; - } - -{xcinside} { - ECHO; - } - -{op_chars} { - ECHO; - } - -\*+ { - ECHO; - } - -{xbstart} { - BEGIN(xb); - ECHO; - } -{quotestop} | -{quotefail} { - yyless(1); - BEGIN(INITIAL); - ECHO; - } -{xhinside} | -{xbinside} { - ECHO; - } -{quotecontinue} | -{quotecontinue} { - ECHO; - } - -{xhstart} { - /* Hexadecimal bit type. - * At some point we should simply pass the string - * forward to the parser and label it there. - * In the meantime, place a leading "x" on the string - * to mark it for the input routine as a hex string. - */ - BEGIN(xh); - ECHO; - } -{quotestop} | -{quotefail} { - yyless(1); - BEGIN(INITIAL); - ECHO; - } - -{xnstart} { - yyless(1); /* eat only 'n' this time */ - ECHO; - } - -{xqstart} { - if (cur_state->std_strings) - BEGIN(xq); - else - BEGIN(xe); - ECHO; - } -{xestart} { - BEGIN(xe); - ECHO; - } -{xusstart} { - BEGIN(xus); - ECHO; - } -{quotestop} | -{quotefail} { - yyless(1); - BEGIN(INITIAL); - ECHO; - } -{quotestop} | -{quotefail} { - /* throw back all but the quote */ - yyless(1); - BEGIN(xusend); - ECHO; - } -{whitespace} { - ECHO; - } -{other} | -{xustop1} { - yyless(0); - BEGIN(INITIAL); - ECHO; - } -{xustop2} { - BEGIN(INITIAL); - ECHO; - } -{xqdouble} { - ECHO; - } -{xqinside} { - ECHO; - } -{xeinside} { - ECHO; - } -{xeunicode} { - ECHO; - } -{xeunicodefail} { - ECHO; - } -{xeescape} { - ECHO; - } -{xeoctesc} { - ECHO; - } -{xehexesc} { - ECHO; - } -{quotecontinue} { - ECHO; - } -. { - /* This is only needed for \ just before EOF */ - ECHO; - } - -{dolqdelim} { - cur_state->dolqstart = pg_strdup(yytext); - BEGIN(xdolq); - ECHO; - } -{dolqfailed} { - /* throw back all but the initial "$" */ - yyless(1); - ECHO; - } -{dolqdelim} { - if (strcmp(yytext, cur_state->dolqstart) == 0) - { - free(cur_state->dolqstart); - cur_state->dolqstart = NULL; - BEGIN(INITIAL); - } - else - { - /* - * When we fail to match $...$ to dolqstart, transfer - * the $... part to the output, but put back the final - * $ for rescanning. Consider $delim$...$junk$delim$ - */ - yyless(yyleng - 1); - } - ECHO; - } -{dolqinside} { - ECHO; - } -{dolqfailed} { - ECHO; - } -. { - /* This is only needed for $ inside the quoted text */ - ECHO; - } - -{xdstart} { - BEGIN(xd); - ECHO; - } -{xuistart} { - BEGIN(xui); - ECHO; - } -{xdstop} { - BEGIN(INITIAL); - ECHO; - } -{dquote} { - yyless(1); - BEGIN(xuiend); - ECHO; - } -{whitespace} { - ECHO; - } -{other} | -{xustop1} { - yyless(0); - BEGIN(INITIAL); - ECHO; - } -{xustop2} { - BEGIN(INITIAL); - ECHO; - } -{xddouble} { - ECHO; - } -{xdinside} { - ECHO; - } - -{xufailed} { - /* throw back all but the initial u/U */ - yyless(1); - ECHO; - } - -{typecast} { - ECHO; - } - -{dot_dot} { - ECHO; - } - -{colon_equals} { - ECHO; - } - -{equals_greater} { - ECHO; - } - -{less_equals} { - ECHO; - } - -{greater_equals} { - ECHO; - } - -{less_greater} { - ECHO; - } - -{not_equals} { - ECHO; - } - - /* - * These rules are specific to psql --- they implement parenthesis - * counting and detection of command-ending semicolon. These must - * appear before the {self} rule so that they take precedence over it. - */ - -"(" { - cur_state->paren_depth++; - ECHO; - } - -")" { - if (cur_state->paren_depth > 0) - cur_state->paren_depth--; - ECHO; - } - -";" { - ECHO; - if (cur_state->paren_depth == 0) - { - /* Terminate lexing temporarily */ - cur_state->start_state = YY_START; - return LEXRES_SEMI; - } - } - - /* - * psql-specific rules to handle backslash commands and variable - * substitution. We want these before {self}, also. - */ - -"\\"[;:] { - /* Force a semicolon or colon into the query buffer */ - psqlscan_emit(cur_state, yytext + 1, 1); - } - -"\\" { - /* Terminate lexing temporarily */ - cur_state->start_state = YY_START; - return LEXRES_BACKSLASH; - } - -:{variable_char}+ { - /* Possible psql variable substitution */ - char *varname; - char *value; - - varname = psqlscan_extract_substring(cur_state, - yytext + 1, - yyleng - 1); - if (cur_state->callbacks->get_variable) - value = cur_state->callbacks->get_variable(varname, - false, - false); - else - value = NULL; - - if (value) - { - /* It is a variable, check for recursion */ - if (var_is_current_source(cur_state, varname)) - { - /* Recursive expansion --- don't go there */ - cur_state->callbacks->write_error("skipping recursive expansion of variable \"%s\"\n", - varname); - /* Instead copy the string as is */ - ECHO; - } - else - { - /* OK, perform substitution */ - psqlscan_push_new_buffer(cur_state, value, varname); - /* yy_scan_string already made buffer active */ - } - free(value); - } - else - { - /* - * if the variable doesn't exist we'll copy the string - * as is - */ - ECHO; - } - - free(varname); - } - -:'{variable_char}+' { - psqlscan_escape_variable(cur_state, yytext, yyleng, false); - } - -:\"{variable_char}+\" { - psqlscan_escape_variable(cur_state, yytext, yyleng, true); - } - - /* - * These rules just avoid the need for scanner backup if one of the - * two rules above fails to match completely. - */ - -:'{variable_char}* { - /* Throw back everything but the colon */ - yyless(1); - ECHO; - } - -:\"{variable_char}* { - /* Throw back everything but the colon */ - yyless(1); - ECHO; - } - - /* - * Back to backend-compatible rules. - */ - -{self} { - ECHO; - } - -{operator} { - /* - * Check for embedded slash-star or dash-dash; those - * are comment starts, so operator must stop there. - * Note that slash-star or dash-dash at the first - * character will match a prior rule, not this one. - */ - int nchars = yyleng; - char *slashstar = strstr(yytext, "/*"); - char *dashdash = strstr(yytext, "--"); - - if (slashstar && dashdash) - { - /* if both appear, take the first one */ - if (slashstar > dashdash) - slashstar = dashdash; - } - else if (!slashstar) - slashstar = dashdash; - if (slashstar) - nchars = slashstar - yytext; - - /* - * For SQL compatibility, '+' and '-' cannot be the - * last char of a multi-char operator unless the operator - * contains chars that are not in SQL operators. - * The idea is to lex '=-' as two operators, but not - * to forbid operator names like '?-' that could not be - * sequences of SQL operators. - */ - while (nchars > 1 && - (yytext[nchars - 1] == '+' || - yytext[nchars - 1] == '-')) - { - int ic; - - for (ic = nchars - 2; ic >= 0; ic--) - { - if (strchr("~!@#^&|`?%", yytext[ic])) - break; - } - if (ic >= 0) - break; /* found a char that makes it OK */ - nchars--; /* else remove the +/-, and check again */ - } - - if (nchars < yyleng) - { - /* Strip the unwanted chars from the token */ - yyless(nchars); - } - ECHO; - } - -{param} { - ECHO; - } - -{integer} { - ECHO; - } -{decimal} { - ECHO; - } -{decimalfail} { - /* throw back the .., and treat as integer */ - yyless(yyleng - 2); - ECHO; - } -{real} { - ECHO; - } -{realfail1} { - /* - * throw back the [Ee], and treat as {decimal}. Note - * that it is possible the input is actually {integer}, - * but since this case will almost certainly lead to a - * syntax error anyway, we don't bother to distinguish. - */ - yyless(yyleng - 1); - ECHO; - } -{realfail2} { - /* throw back the [Ee][+-], and proceed as above */ - yyless(yyleng - 2); - ECHO; - } - - -{identifier} { - ECHO; - } - -{other} { - ECHO; - } - -<> { - if (cur_state->buffer_stack == NULL) - { - cur_state->start_state = YY_START; - return LEXRES_EOL; /* end of input reached */ - } - - /* - * We were expanding a variable, so pop the inclusion - * stack and keep lexing - */ - psqlscan_pop_buffer_stack(cur_state); - psqlscan_select_top_buffer(cur_state); - } - -%% - -/* - * Create a lexer working state struct. - * - * callbacks is a struct of function pointers that encapsulate some - * behavior we need from the surrounding program. This struct must - * remain valid for the lifespan of the PsqlScanState. - */ -PsqlScanState -psql_scan_create(const PsqlScanCallbacks *callbacks) -{ - PsqlScanState state; - - state = (PsqlScanStateData *) pg_malloc0(sizeof(PsqlScanStateData)); - - state->callbacks = callbacks; - - yylex_init(&state->scanner); - - yyset_extra(state, state->scanner); - - psql_scan_reset(state); - - return state; -} - -/* - * Destroy a lexer working state struct, releasing all resources. - */ -void -psql_scan_destroy(PsqlScanState state) -{ - psql_scan_finish(state); - - psql_scan_reset(state); - - yylex_destroy(state->scanner); - - free(state); -} - -/* - * Set up to perform lexing of the given input line. - * - * The text at *line, extending for line_len bytes, will be scanned by - * subsequent calls to the psql_scan routines. psql_scan_finish should - * be called when scanning is complete. Note that the lexer retains - * a pointer to the storage at *line --- this string must not be altered - * or freed until after psql_scan_finish is called. - * - * encoding is the libpq identifier for the character encoding in use, - * and std_strings says whether standard_conforming_strings is on. - */ -void -psql_scan_setup(PsqlScanState state, - const char *line, int line_len, - int encoding, bool std_strings) -{ - /* Mustn't be scanning already */ - Assert(state->scanbufhandle == NULL); - Assert(state->buffer_stack == NULL); - - /* Do we need to hack the character set encoding? */ - state->encoding = encoding; - state->safe_encoding = pg_valid_server_encoding_id(encoding); - - /* Save standard-strings flag as well */ - state->std_strings = std_strings; - - /* Set up flex input buffer with appropriate translation and padding */ - state->scanbufhandle = psqlscan_prepare_buffer(state, line, line_len, - &state->scanbuf); - state->scanline = line; - - /* Set lookaside data in case we have to map unsafe encoding */ - state->curline = state->scanbuf; - state->refline = state->scanline; -} - -/* - * Do lexical analysis of SQL command text. - * - * The text previously passed to psql_scan_setup is scanned, and appended - * (possibly with transformation) to query_buf. - * - * The return value indicates the condition that stopped scanning: - * - * PSCAN_SEMICOLON: found a command-ending semicolon. (The semicolon is - * transferred to query_buf.) The command accumulated in query_buf should - * be executed, then clear query_buf and call again to scan the remainder - * of the line. - * - * PSCAN_BACKSLASH: found a backslash that starts a special command. - * Any previous data on the line has been transferred to query_buf. - * The caller will typically next apply a separate flex lexer to scan - * the special command. - * - * PSCAN_INCOMPLETE: the end of the line was reached, but we have an - * incomplete SQL command. *prompt is set to the appropriate prompt type. - * - * PSCAN_EOL: the end of the line was reached, and there is no lexical - * reason to consider the command incomplete. The caller may or may not - * choose to send it. *prompt is set to the appropriate prompt type if - * the caller chooses to collect more input. - * - * In the PSCAN_INCOMPLETE and PSCAN_EOL cases, psql_scan_finish() should - * be called next, then the cycle may be repeated with a fresh input line. - * - * In all cases, *prompt is set to an appropriate prompt type code for the - * next line-input operation. - */ -PsqlScanResult -psql_scan(PsqlScanState state, - PQExpBuffer query_buf, - promptStatus_t *prompt) -{ - PsqlScanResult result; - int lexresult; - - /* Must be scanning already */ - Assert(state->scanbufhandle != NULL); - - /* Set current output target */ - state->output_buf = query_buf; - - /* Set input source */ - if (state->buffer_stack != NULL) - yy_switch_to_buffer(state->buffer_stack->buf, state->scanner); - else - yy_switch_to_buffer(state->scanbufhandle, state->scanner); - - /* And lex. */ - lexresult = yylex(NULL, state->scanner); - - /* - * Check termination state and return appropriate result info. - */ - switch (lexresult) - { - case LEXRES_EOL: /* end of input */ - switch (state->start_state) - { - case INITIAL: - case xuiend: /* we treat these like INITIAL */ - case xusend: - if (state->paren_depth > 0) - { - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_PAREN; - } - else if (query_buf->len > 0) - { - result = PSCAN_EOL; - *prompt = PROMPT_CONTINUE; - } - else - { - /* never bother to send an empty buffer */ - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_READY; - } - break; - case xb: - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_SINGLEQUOTE; - break; - case xc: - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_COMMENT; - break; - case xd: - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_DOUBLEQUOTE; - break; - case xh: - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_SINGLEQUOTE; - break; - case xe: - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_SINGLEQUOTE; - break; - case xq: - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_SINGLEQUOTE; - break; - case xdolq: - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_DOLLARQUOTE; - break; - case xui: - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_DOUBLEQUOTE; - break; - case xus: - result = PSCAN_INCOMPLETE; - *prompt = PROMPT_SINGLEQUOTE; - break; - default: - /* can't get here */ - fprintf(stderr, "invalid YY_START\n"); - exit(1); - } - break; - case LEXRES_SEMI: /* semicolon */ - result = PSCAN_SEMICOLON; - *prompt = PROMPT_READY; - break; - case LEXRES_BACKSLASH: /* backslash */ - result = PSCAN_BACKSLASH; - *prompt = PROMPT_READY; - break; - default: - /* can't get here */ - fprintf(stderr, "invalid yylex result\n"); - exit(1); - } - - return result; -} - -/* - * Clean up after scanning a string. This flushes any unread input and - * releases resources (but not the PsqlScanState itself). Note however - * that this does not reset the lexer scan state; that can be done by - * psql_scan_reset(), which is an orthogonal operation. - * - * It is legal to call this when not scanning anything (makes it easier - * to deal with error recovery). - */ -void -psql_scan_finish(PsqlScanState state) -{ - /* Drop any incomplete variable expansions. */ - while (state->buffer_stack != NULL) - psqlscan_pop_buffer_stack(state); - - /* Done with the outer scan buffer, too */ - if (state->scanbufhandle) - yy_delete_buffer(state->scanbufhandle, state->scanner); - state->scanbufhandle = NULL; - if (state->scanbuf) - free(state->scanbuf); - state->scanbuf = NULL; -} - -/* - * Reset lexer scanning state to start conditions. This is appropriate - * for executing \r psql commands (or any other time that we discard the - * prior contents of query_buf). It is not, however, necessary to do this - * when we execute and clear the buffer after getting a PSCAN_SEMICOLON or - * PSCAN_EOL scan result, because the scan state must be INITIAL when those - * conditions are returned. - * - * Note that this is unrelated to flushing unread input; that task is - * done by psql_scan_finish(). - */ -void -psql_scan_reset(PsqlScanState state) -{ - state->start_state = INITIAL; - state->paren_depth = 0; - state->xcdepth = 0; /* not really necessary */ - if (state->dolqstart) - free(state->dolqstart); - state->dolqstart = NULL; -} - -/* - * Reselect this lexer (psqlscan.l) after using another one. - * - * Currently and for foreseeable uses, it's sufficient to reset to INITIAL - * state, because we'd never switch to another lexer in a different state. - * However, we don't want to reset e.g. paren_depth, so this can't be - * the same as psql_scan_reset(). - * - * Note: psql setjmp error recovery just calls psql_scan_reset(), so that - * must be a superset of this. - * - * Note: it seems likely that other lexers could just assign INITIAL for - * themselves, since that probably has the value zero in every flex-generated - * lexer. But let's not assume that. - */ -void -psql_scan_reselect_sql_lexer(PsqlScanState state) -{ - state->start_state = INITIAL; -} - -/* - * Return true if lexer is currently in an "inside quotes" state. - * - * This is pretty grotty but is needed to preserve the old behavior - * that mainloop.c drops blank lines not inside quotes without even - * echoing them. - */ -bool -psql_scan_in_quote(PsqlScanState state) -{ - return state->start_state != INITIAL; -} - -/* - * Push the given string onto the stack of stuff to scan. - * - * NOTE SIDE EFFECT: the new buffer is made the active flex input buffer. - */ -void -psqlscan_push_new_buffer(PsqlScanState state, const char *newstr, - const char *varname) -{ - StackElem *stackelem; - - stackelem = (StackElem *) pg_malloc(sizeof(StackElem)); - - /* - * In current usage, the passed varname points at the current flex input - * buffer; we must copy it before calling psqlscan_prepare_buffer() - * because that will change the buffer state. - */ - stackelem->varname = varname ? pg_strdup(varname) : NULL; - - stackelem->buf = psqlscan_prepare_buffer(state, newstr, strlen(newstr), - &stackelem->bufstring); - state->curline = stackelem->bufstring; - if (state->safe_encoding) - { - stackelem->origstring = NULL; - state->refline = stackelem->bufstring; - } - else - { - stackelem->origstring = pg_strdup(newstr); - state->refline = stackelem->origstring; - } - stackelem->next = state->buffer_stack; - state->buffer_stack = stackelem; -} - -/* - * Pop the topmost buffer stack item (there must be one!) - * - * NB: after this, the flex input state is unspecified; caller must - * switch to an appropriate buffer to continue lexing. - * See psqlscan_select_top_buffer(). - */ -void -psqlscan_pop_buffer_stack(PsqlScanState state) -{ - StackElem *stackelem = state->buffer_stack; - - state->buffer_stack = stackelem->next; - yy_delete_buffer(stackelem->buf, state->scanner); - free(stackelem->bufstring); - if (stackelem->origstring) - free(stackelem->origstring); - if (stackelem->varname) - free(stackelem->varname); - free(stackelem); -} - -/* - * Select the topmost surviving buffer as the active input. - */ -void -psqlscan_select_top_buffer(PsqlScanState state) -{ - StackElem *stackelem = state->buffer_stack; - - if (stackelem != NULL) - { - yy_switch_to_buffer(stackelem->buf, state->scanner); - state->curline = stackelem->bufstring; - state->refline = stackelem->origstring ? stackelem->origstring : stackelem->bufstring; - } - else - { - yy_switch_to_buffer(state->scanbufhandle, state->scanner); - state->curline = state->scanbuf; - state->refline = state->scanline; - } -} - -/* - * Check if specified variable name is the source for any string - * currently being scanned - */ -static bool -var_is_current_source(PsqlScanState state, const char *varname) -{ - StackElem *stackelem; - - for (stackelem = state->buffer_stack; - stackelem != NULL; - stackelem = stackelem->next) - { - if (stackelem->varname && strcmp(stackelem->varname, varname) == 0) - return true; - } - return false; -} - -/* - * Set up a flex input buffer to scan the given data. We always make a - * copy of the data. If working in an unsafe encoding, the copy has - * multibyte sequences replaced by FFs to avoid fooling the lexer rules. - * - * NOTE SIDE EFFECT: the new buffer is made the active flex input buffer. - */ -YY_BUFFER_STATE -psqlscan_prepare_buffer(PsqlScanState state, const char *txt, int len, - char **txtcopy) -{ - char *newtxt; - - /* Flex wants two \0 characters after the actual data */ - newtxt = pg_malloc(len + 2); - *txtcopy = newtxt; - newtxt[len] = newtxt[len + 1] = YY_END_OF_BUFFER_CHAR; - - if (state->safe_encoding) - memcpy(newtxt, txt, len); - else - { - /* Gotta do it the hard way */ - int i = 0; - - while (i < len) - { - int thislen = PQmblen(txt + i, state->encoding); - - /* first byte should always be okay... */ - newtxt[i] = txt[i]; - i++; - while (--thislen > 0 && i < len) - newtxt[i++] = (char) 0xFF; - } - } - - return yy_scan_buffer(newtxt, len + 2, state->scanner); -} - -/* - * psqlscan_emit() --- body for ECHO macro - * - * NB: this must be used for ALL and ONLY the text copied from the flex - * input data. If you pass it something that is not part of the yytext - * string, you are making a mistake. Internally generated text can be - * appended directly to state->output_buf. - */ -void -psqlscan_emit(PsqlScanState state, const char *txt, int len) -{ - PQExpBuffer output_buf = state->output_buf; - - if (state->safe_encoding) - appendBinaryPQExpBuffer(output_buf, txt, len); - else - { - /* Gotta do it the hard way */ - const char *reference = state->refline; - int i; - - reference += (txt - state->curline); - - for (i = 0; i < len; i++) - { - char ch = txt[i]; - - if (ch == (char) 0xFF) - ch = reference[i]; - appendPQExpBufferChar(output_buf, ch); - } - } -} - -/* - * psqlscan_extract_substring --- fetch value of (part of) the current token - * - * This is like psqlscan_emit(), except that the data is returned as a - * malloc'd string rather than being pushed directly to state->output_buf. - */ -char * -psqlscan_extract_substring(PsqlScanState state, const char *txt, int len) -{ - char *result = (char *) pg_malloc(len + 1); - - if (state->safe_encoding) - memcpy(result, txt, len); - else - { - /* Gotta do it the hard way */ - const char *reference = state->refline; - int i; - - reference += (txt - state->curline); - - for (i = 0; i < len; i++) - { - char ch = txt[i]; - - if (ch == (char) 0xFF) - ch = reference[i]; - result[i] = ch; - } - } - result[len] = '\0'; - return result; -} - -/* - * psqlscan_escape_variable --- process :'VARIABLE' or :"VARIABLE" - * - * If the variable name is found, escape its value using the appropriate - * quoting method and emit the value to output_buf. (Since the result is - * surely quoted, there is never any reason to rescan it.) If we don't - * find the variable or escaping fails, emit the token as-is. - */ -void -psqlscan_escape_variable(PsqlScanState state, const char *txt, int len, - bool as_ident) -{ - char *varname; - char *value; - - /* Variable lookup. */ - varname = psqlscan_extract_substring(state, txt + 2, len - 3); - if (state->callbacks->get_variable) - value = state->callbacks->get_variable(varname, true, as_ident); - else - value = NULL; - free(varname); - - if (value) - { - /* Emit the suitably-escaped value */ - appendPQExpBufferStr(state->output_buf, value); - free(value); - } - else - { - /* Emit original token as-is */ - psqlscan_emit(state, txt, len); - } -} diff --git a/src/bin/psql/psqlscan_int.h b/src/bin/psql/psqlscan_int.h deleted file mode 100644 index cdbf85d5b24..00000000000 --- a/src/bin/psql/psqlscan_int.h +++ /dev/null @@ -1,129 +0,0 @@ -/* - * psqlscan_int.h - * lexical scanner internal declarations - * - * This file declares the PsqlScanStateData structure used by psqlscan.l - * and shared by other lexers compatible with it, such as psqlscanslash.l. - * - * One difficult aspect of this code is that we need to work in multibyte - * encodings that are not ASCII-safe. A "safe" encoding is one in which each - * byte of a multibyte character has the high bit set (it's >= 0x80). Since - * all our lexing rules treat all high-bit-set characters alike, we don't - * really need to care whether such a byte is part of a sequence or not. - * In an "unsafe" encoding, we still expect the first byte of a multibyte - * sequence to be >= 0x80, but later bytes might not be. If we scan such - * a sequence as-is, the lexing rules could easily be fooled into matching - * such bytes to ordinary ASCII characters. Our solution for this is to - * substitute 0xFF for each non-first byte within the data presented to flex. - * The flex rules will then pass the FF's through unmolested. The - * psqlscan_emit() subroutine is responsible for looking back to the original - * string and replacing FF's with the corresponding original bytes. - * - * Another interesting thing we do here is scan different parts of the same - * input with physically separate flex lexers (ie, lexers written in separate - * .l files). We can get away with this because the only part of the - * persistent state of a flex lexer that depends on its parsing rule tables - * is the start state number, which is easy enough to manage --- usually, - * in fact, we just need to set it to INITIAL when changing lexers. But to - * make that work at all, we must use re-entrant lexers, so that all the - * relevant state is in the yyscanner_t attached to the PsqlScanState; - * if we were using lexers with separate static state we would soon end up - * with dangling buffer pointers in one or the other. Also note that this - * is unlikely to work very nicely if the lexers aren't all built with the - * same flex version. - * - * Copyright (c) 2000-2016, PostgreSQL Global Development Group - * - * src/bin/psql/psqlscan_int.h - */ -#ifndef PSQLSCAN_INT_H -#define PSQLSCAN_INT_H - -#include "psqlscan.h" - -/* This is just to allow this file to be compilable standalone */ -#ifndef YY_TYPEDEF_YY_BUFFER_STATE -#define YY_TYPEDEF_YY_BUFFER_STATE -typedef struct yy_buffer_state *YY_BUFFER_STATE; -#endif - -/* - * We use a stack of flex buffers to handle substitution of psql variables. - * Each stacked buffer contains the as-yet-unread text from one psql variable. - * When we pop the stack all the way, we resume reading from the outer buffer - * identified by scanbufhandle. - */ -typedef struct StackElem -{ - YY_BUFFER_STATE buf; /* flex input control structure */ - char *bufstring; /* data actually being scanned by flex */ - char *origstring; /* copy of original data, if needed */ - char *varname; /* name of variable providing data, or NULL */ - struct StackElem *next; -} StackElem; - -/* - * All working state of the lexer must be stored in PsqlScanStateData - * between calls. This allows us to have multiple open lexer operations, - * which is needed for nested include files. The lexer itself is not - * recursive, but it must be re-entrant. - */ -typedef struct PsqlScanStateData -{ - yyscan_t scanner; /* Flex's state for this PsqlScanState */ - - PQExpBuffer output_buf; /* current output buffer */ - - StackElem *buffer_stack; /* stack of variable expansion buffers */ - - /* - * These variables always refer to the outer buffer, never to any stacked - * variable-expansion buffer. - */ - YY_BUFFER_STATE scanbufhandle; - char *scanbuf; /* start of outer-level input buffer */ - const char *scanline; /* current input line at outer level */ - - /* safe_encoding, curline, refline are used by emit() to replace FFs */ - int encoding; /* encoding being used now */ - bool safe_encoding; /* is current encoding "safe"? */ - bool std_strings; /* are string literals standard? */ - const char *curline; /* actual flex input string for cur buf */ - const char *refline; /* original data for cur buffer */ - - /* - * All this state lives across successive input lines, until explicitly - * reset by psql_scan_reset. start_state is adopted by yylex() on entry, - * and updated with its finishing state on exit. - */ - int start_state; /* yylex's starting/finishing state */ - int paren_depth; /* depth of nesting in parentheses */ - int xcdepth; /* depth of nesting in slash-star comments */ - char *dolqstart; /* current $foo$ quote start string */ - - /* - * Callback functions provided by the program making use of the lexer. - */ - const PsqlScanCallbacks *callbacks; -} PsqlScanStateData; - - -/* - * Functions exported by psqlscan.l, but only meant for use within - * compatible lexers. - */ -extern void psqlscan_push_new_buffer(PsqlScanState state, - const char *newstr, const char *varname); -extern void psqlscan_pop_buffer_stack(PsqlScanState state); -extern void psqlscan_select_top_buffer(PsqlScanState state); -extern YY_BUFFER_STATE psqlscan_prepare_buffer(PsqlScanState state, - const char *txt, int len, - char **txtcopy); -extern void psqlscan_emit(PsqlScanState state, const char *txt, int len); -extern char *psqlscan_extract_substring(PsqlScanState state, - const char *txt, int len); -extern void psqlscan_escape_variable(PsqlScanState state, - const char *txt, int len, - bool as_ident); - -#endif /* PSQLSCAN_INT_H */ diff --git a/src/bin/psql/psqlscanslash.h b/src/bin/psql/psqlscanslash.h index abc3700d001..48553647a90 100644 --- a/src/bin/psql/psqlscanslash.h +++ b/src/bin/psql/psqlscanslash.h @@ -8,7 +8,7 @@ #ifndef PSQLSCANSLASH_H #define PSQLSCANSLASH_H -#include "psqlscan.h" +#include "fe_utils/psqlscan.h" /* Different ways for scan_slash_option to handle parameter words */ diff --git a/src/bin/psql/psqlscanslash.l b/src/bin/psql/psqlscanslash.l index a89ce15ad4c..e3e0db3b2fb 100644 --- a/src/bin/psql/psqlscanslash.l +++ b/src/bin/psql/psqlscanslash.l @@ -6,7 +6,7 @@ * * XXX Avoid creating backtracking cases --- see the backend lexer for info. * - * See psqlscan_int.h for additional commentary. + * See fe_utils/psqlscan_int.h for additional commentary. * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -24,7 +24,7 @@ } %{ -#include "psqlscan_int.h" +#include "fe_utils/psqlscan_int.h" /* * We must have a typedef YYSTYPE for yylex's first argument, but this lexer diff --git a/src/fe_utils/.gitignore b/src/fe_utils/.gitignore new file mode 100644 index 00000000000..37f5f7514df --- /dev/null +++ b/src/fe_utils/.gitignore @@ -0,0 +1 @@ +/psqlscan.c diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile index 3223f1f26c7..9da03b196b5 100644 --- a/src/fe_utils/Makefile +++ b/src/fe_utils/Makefile @@ -1,11 +1,13 @@ #------------------------------------------------------------------------- # -# Makefile -# Makefile for src/fe_utils +# Makefile for src/fe_utils # # This makefile generates a static library, libpgfeutils.a, # for use by client applications # +# Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# # IDENTIFICATION # src/fe_utils/Makefile # @@ -17,7 +19,7 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS) -OBJS = mbprint.o print.o simple_list.o string_utils.o +OBJS = mbprint.o print.o psqlscan.o simple_list.o string_utils.o all: libpgfeutils.a @@ -25,6 +27,16 @@ libpgfeutils.a: $(OBJS) rm -f $@ $(AR) $(AROPT) $@ $^ +psqlscan.c: FLEXFLAGS = -Cfe -p -p +psqlscan.c: FLEX_NO_BACKUP=yes + +# Latest flex causes warnings in this file. +ifeq ($(GCC),yes) +psqlscan.o: CFLAGS += -Wno-error +endif + +distprep: psqlscan.c + # libpgfeutils could be useful to contrib, so install it install: all installdirs $(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(libdir)/libpgfeutils.a' @@ -35,5 +47,10 @@ installdirs: uninstall: rm -f '$(DESTDIR)$(libdir)/libpgfeutils.a' -clean distclean maintainer-clean: - rm -f libpgfeutils.a $(OBJS) +clean distclean: + rm -f libpgfeutils.a $(OBJS) lex.backup + +# psqlscan.c is supposed to be in the distribution tarball, +# so do not clean it in the clean/distclean rules +maintainer-clean: distclean + rm -f psqlscan.c diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l new file mode 100644 index 00000000000..55067b450c2 --- /dev/null +++ b/src/fe_utils/psqlscan.l @@ -0,0 +1,1428 @@ +%top{ +/*------------------------------------------------------------------------- + * + * psqlscan.l + * lexical scanner for SQL commands + * + * This lexer used to be part of psql, and that heritage is reflected in + * the file name as well as function and typedef names, though it can now + * be used by other frontend programs as well. It's also possible to extend + * this lexer with a compatible add-on lexer to handle program-specific + * backslash commands. + * + * This code is mainly concerned with determining where the end of a SQL + * statement is: we are looking for semicolons that are not within quotes, + * comments, or parentheses. The most reliable way to handle this is to + * borrow the backend's flex lexer rules, lock, stock, and barrel. The rules + * below are (except for a few) the same as the backend's, but their actions + * are just ECHO whereas the backend's actions generally do other things. + * + * XXX The rules in this file must be kept in sync with the backend lexer!!! + * + * XXX Avoid creating backtracking cases --- see the backend lexer for info. + * + * See psqlscan_int.h for additional commentary. + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/fe_utils/psqlscan.l + * + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" + +#include "fe_utils/psqlscan.h" + +#include "libpq-fe.h" +} + +%{ +#include "fe_utils/psqlscan_int.h" + +/* + * We must have a typedef YYSTYPE for yylex's first argument, but this lexer + * doesn't presently make use of that argument, so just declare it as int. + */ +typedef int YYSTYPE; + +/* + * Set the type of yyextra; we use it as a pointer back to the containing + * PsqlScanState. + */ +#define YY_EXTRA_TYPE PsqlScanState + + +/* Return values from yylex() */ +#define LEXRES_EOL 0 /* end of input */ +#define LEXRES_SEMI 1 /* command-terminating semicolon found */ +#define LEXRES_BACKSLASH 2 /* backslash command start */ + + +#define ECHO psqlscan_emit(cur_state, yytext, yyleng) + +/* + * Work around a bug in flex 2.5.35: it emits a couple of functions that + * it forgets to emit declarations for. Since we use -Wmissing-prototypes, + * this would cause warnings. Providing our own declarations should be + * harmless even when the bug gets fixed. + */ +extern int psql_yyget_column(yyscan_t yyscanner); +extern void psql_yyset_column(int column_no, yyscan_t yyscanner); + +%} + +%option reentrant +%option bison-bridge +%option 8bit +%option never-interactive +%option nodefault +%option noinput +%option nounput +%option noyywrap +%option warn +%option prefix="psql_yy" + +/* + * All of the following definitions and rules should exactly match + * src/backend/parser/scan.l so far as the flex patterns are concerned. + * The rule bodies are just ECHO as opposed to what the backend does, + * however. (But be sure to duplicate code that affects the lexing process, + * such as BEGIN() and yyless().) Also, psqlscan uses a single <> rule + * whereas scan.l has a separate one for each exclusive state. + */ + +/* + * OK, here is a short description of lex/flex rules behavior. + * The longest pattern which matches an input string is always chosen. + * For equal-length patterns, the first occurring in the rules list is chosen. + * INITIAL is the starting state, to which all non-conditional rules apply. + * Exclusive states change parsing rules while the state is active. When in + * an exclusive state, only those rules defined for that state apply. + * + * We use exclusive states for quoted strings, extended comments, + * and to eliminate parsing troubles for numeric strings. + * Exclusive states: + * bit string literal + * extended C-style comments + * delimited identifiers (double-quoted identifiers) + * hexadecimal numeric string + * standard quoted strings + * extended quoted strings (support backslash escape sequences) + * $foo$ quoted strings + * quoted identifier with Unicode escapes + * end of a quoted identifier with Unicode escapes, UESCAPE can follow + * quoted string with Unicode escapes + * end of a quoted string with Unicode escapes, UESCAPE can follow + * + * Note: we intentionally don't mimic the backend's state; we have + * no need to distinguish it from state, and no good way to get out + * of it in error cases. The backend just throws yyerror() in those + * cases, but that's not an option here. + */ + +%x xb +%x xc +%x xd +%x xh +%x xe +%x xq +%x xdolq +%x xui +%x xuiend +%x xus +%x xusend + +/* + * In order to make the world safe for Windows and Mac clients as well as + * Unix ones, we accept either \n or \r as a newline. A DOS-style \r\n + * sequence will be seen as two successive newlines, but that doesn't cause + * any problems. Comments that start with -- and extend to the next + * newline are treated as equivalent to a single whitespace character. + * + * NOTE a fine point: if there is no newline following --, we will absorb + * everything to the end of the input as a comment. This is correct. Older + * versions of Postgres failed to recognize -- as a comment if the input + * did not end with a newline. + * + * XXX perhaps \f (formfeed) should be treated as a newline as well? + * + * XXX if you change the set of whitespace characters, fix scanner_isspace() + * to agree, and see also the plpgsql lexer. + */ + +space [ \t\n\r\f] +horiz_space [ \t\f] +newline [\n\r] +non_newline [^\n\r] + +comment ("--"{non_newline}*) + +whitespace ({space}+|{comment}) + +/* + * SQL requires at least one newline in the whitespace separating + * string literals that are to be concatenated. Silly, but who are we + * to argue? Note that {whitespace_with_newline} should not have * after + * it, whereas {whitespace} should generally have a * after it... + */ + +special_whitespace ({space}+|{comment}{newline}) +horiz_whitespace ({horiz_space}|{comment}) +whitespace_with_newline ({horiz_whitespace}*{newline}{special_whitespace}*) + +/* + * To ensure that {quotecontinue} can be scanned without having to back up + * if the full pattern isn't matched, we include trailing whitespace in + * {quotestop}. This matches all cases where {quotecontinue} fails to match, + * except for {quote} followed by whitespace and just one "-" (not two, + * which would start a {comment}). To cover that we have {quotefail}. + * The actions for {quotestop} and {quotefail} must throw back characters + * beyond the quote proper. + */ +quote ' +quotestop {quote}{whitespace}* +quotecontinue {quote}{whitespace_with_newline}{quote} +quotefail {quote}{whitespace}*"-" + +/* Bit string + * It is tempting to scan the string for only those characters + * which are allowed. However, this leads to silently swallowed + * characters if illegal characters are included in the string. + * For example, if xbinside is [01] then B'ABCD' is interpreted + * as a zero-length string, and the ABCD' is lost! + * Better to pass the string forward and let the input routines + * validate the contents. + */ +xbstart [bB]{quote} +xbinside [^']* + +/* Hexadecimal number */ +xhstart [xX]{quote} +xhinside [^']* + +/* National character */ +xnstart [nN]{quote} + +/* Quoted string that allows backslash escapes */ +xestart [eE]{quote} +xeinside [^\\']+ +xeescape [\\][^0-7] +xeoctesc [\\][0-7]{1,3} +xehexesc [\\]x[0-9A-Fa-f]{1,2} +xeunicode [\\](u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8}) +xeunicodefail [\\](u[0-9A-Fa-f]{0,3}|U[0-9A-Fa-f]{0,7}) + +/* Extended quote + * xqdouble implements embedded quote, '''' + */ +xqstart {quote} +xqdouble {quote}{quote} +xqinside [^']+ + +/* $foo$ style quotes ("dollar quoting") + * The quoted string starts with $foo$ where "foo" is an optional string + * in the form of an identifier, except that it may not contain "$", + * and extends to the first occurrence of an identical string. + * There is *no* processing of the quoted text. + * + * {dolqfailed} is an error rule to avoid scanner backup when {dolqdelim} + * fails to match its trailing "$". + */ +dolq_start [A-Za-z\200-\377_] +dolq_cont [A-Za-z\200-\377_0-9] +dolqdelim \$({dolq_start}{dolq_cont}*)?\$ +dolqfailed \${dolq_start}{dolq_cont}* +dolqinside [^$]+ + +/* Double quote + * Allows embedded spaces and other special characters into identifiers. + */ +dquote \" +xdstart {dquote} +xdstop {dquote} +xddouble {dquote}{dquote} +xdinside [^"]+ + +/* Unicode escapes */ +uescape [uU][eE][sS][cC][aA][pP][eE]{whitespace}*{quote}[^']{quote} +/* error rule to avoid backup */ +uescapefail [uU][eE][sS][cC][aA][pP][eE]{whitespace}*"-"|[uU][eE][sS][cC][aA][pP][eE]{whitespace}*{quote}[^']|[uU][eE][sS][cC][aA][pP][eE]{whitespace}*{quote}|[uU][eE][sS][cC][aA][pP][eE]{whitespace}*|[uU][eE][sS][cC][aA][pP]|[uU][eE][sS][cC][aA]|[uU][eE][sS][cC]|[uU][eE][sS]|[uU][eE]|[uU] + +/* Quoted identifier with Unicode escapes */ +xuistart [uU]&{dquote} + +/* Quoted string with Unicode escapes */ +xusstart [uU]&{quote} + +/* Optional UESCAPE after a quoted string or identifier with Unicode escapes. */ +xustop1 {uescapefail}? +xustop2 {uescape} + +/* error rule to avoid backup */ +xufailed [uU]& + + +/* C-style comments + * + * The "extended comment" syntax closely resembles allowable operator syntax. + * The tricky part here is to get lex to recognize a string starting with + * slash-star as a comment, when interpreting it as an operator would produce + * a longer match --- remember lex will prefer a longer match! Also, if we + * have something like plus-slash-star, lex will think this is a 3-character + * operator whereas we want to see it as a + operator and a comment start. + * The solution is two-fold: + * 1. append {op_chars}* to xcstart so that it matches as much text as + * {operator} would. Then the tie-breaker (first matching rule of same + * length) ensures xcstart wins. We put back the extra stuff with yyless() + * in case it contains a star-slash that should terminate the comment. + * 2. In the operator rule, check for slash-star within the operator, and + * if found throw it back with yyless(). This handles the plus-slash-star + * problem. + * Dash-dash comments have similar interactions with the operator rule. + */ +xcstart \/\*{op_chars}* +xcstop \*+\/ +xcinside [^*/]+ + +digit [0-9] +ident_start [A-Za-z\200-\377_] +ident_cont [A-Za-z\200-\377_0-9\$] + +identifier {ident_start}{ident_cont}* + +/* Assorted special-case operators and operator-like tokens */ +typecast "::" +dot_dot \.\. +colon_equals ":=" +equals_greater "=>" +less_equals "<=" +greater_equals ">=" +less_greater "<>" +not_equals "!=" + +/* + * "self" is the set of chars that should be returned as single-character + * tokens. "op_chars" is the set of chars that can make up "Op" tokens, + * which can be one or more characters long (but if a single-char token + * appears in the "self" set, it is not to be returned as an Op). Note + * that the sets overlap, but each has some chars that are not in the other. + * + * If you change either set, adjust the character lists appearing in the + * rule for "operator"! + */ +self [,()\[\].;\:\+\-\*\/\%\^\<\>\=] +op_chars [\~\!\@\#\^\&\|\`\?\+\-\*\/\%\<\>\=] +operator {op_chars}+ + +/* we no longer allow unary minus in numbers. + * instead we pass it separately to parser. there it gets + * coerced via doNegate() -- Leon aug 20 1999 + * + * {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot, 10. + * + * {realfail1} and {realfail2} are added to prevent the need for scanner + * backup when the {real} rule fails to match completely. + */ + +integer {digit}+ +decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*)) +decimalfail {digit}+\.\. +real ({integer}|{decimal})[Ee][-+]?{digit}+ +realfail1 ({integer}|{decimal})[Ee] +realfail2 ({integer}|{decimal})[Ee][-+] + +param \${integer} + +/* psql-specific: characters allowed in variable names */ +variable_char [A-Za-z\200-\377_0-9] + +other . + +/* + * Dollar quoted strings are totally opaque, and no escaping is done on them. + * Other quoted strings must allow some special characters such as single-quote + * and newline. + * Embedded single-quotes are implemented both in the SQL standard + * style of two adjacent single quotes "''" and in the Postgres/Java style + * of escaped-quote "\'". + * Other embedded escaped characters are matched explicitly and the leading + * backslash is dropped from the string. + * Note that xcstart must appear before operator, as explained above! + * Also whitespace (comment) must appear before operator. + */ + +%% + +%{ + /* Declare some local variables inside yylex(), for convenience */ + PsqlScanState cur_state = yyextra; + PQExpBuffer output_buf = cur_state->output_buf; + + /* + * Force flex into the state indicated by start_state. This has a + * couple of purposes: it lets some of the functions below set a new + * starting state without ugly direct access to flex variables, and it + * allows us to transition from one flex lexer to another so that we + * can lex different parts of the source string using separate lexers. + */ + BEGIN(cur_state->start_state); +%} + +{whitespace} { + /* + * Note that the whitespace rule includes both true + * whitespace and single-line ("--" style) comments. + * We suppress whitespace at the start of the query + * buffer. We also suppress all single-line comments, + * which is pretty dubious but is the historical + * behavior. + */ + if (!(output_buf->len == 0 || yytext[0] == '-')) + ECHO; + } + +{xcstart} { + cur_state->xcdepth = 0; + BEGIN(xc); + /* Put back any characters past slash-star; see above */ + yyless(2); + ECHO; + } + +{xcstart} { + cur_state->xcdepth++; + /* Put back any characters past slash-star; see above */ + yyless(2); + ECHO; + } + +{xcstop} { + if (cur_state->xcdepth <= 0) + BEGIN(INITIAL); + else + cur_state->xcdepth--; + ECHO; + } + +{xcinside} { + ECHO; + } + +{op_chars} { + ECHO; + } + +\*+ { + ECHO; + } + +{xbstart} { + BEGIN(xb); + ECHO; + } +{quotestop} | +{quotefail} { + yyless(1); + BEGIN(INITIAL); + ECHO; + } +{xhinside} | +{xbinside} { + ECHO; + } +{quotecontinue} | +{quotecontinue} { + ECHO; + } + +{xhstart} { + /* Hexadecimal bit type. + * At some point we should simply pass the string + * forward to the parser and label it there. + * In the meantime, place a leading "x" on the string + * to mark it for the input routine as a hex string. + */ + BEGIN(xh); + ECHO; + } +{quotestop} | +{quotefail} { + yyless(1); + BEGIN(INITIAL); + ECHO; + } + +{xnstart} { + yyless(1); /* eat only 'n' this time */ + ECHO; + } + +{xqstart} { + if (cur_state->std_strings) + BEGIN(xq); + else + BEGIN(xe); + ECHO; + } +{xestart} { + BEGIN(xe); + ECHO; + } +{xusstart} { + BEGIN(xus); + ECHO; + } +{quotestop} | +{quotefail} { + yyless(1); + BEGIN(INITIAL); + ECHO; + } +{quotestop} | +{quotefail} { + /* throw back all but the quote */ + yyless(1); + BEGIN(xusend); + ECHO; + } +{whitespace} { + ECHO; + } +{other} | +{xustop1} { + yyless(0); + BEGIN(INITIAL); + ECHO; + } +{xustop2} { + BEGIN(INITIAL); + ECHO; + } +{xqdouble} { + ECHO; + } +{xqinside} { + ECHO; + } +{xeinside} { + ECHO; + } +{xeunicode} { + ECHO; + } +{xeunicodefail} { + ECHO; + } +{xeescape} { + ECHO; + } +{xeoctesc} { + ECHO; + } +{xehexesc} { + ECHO; + } +{quotecontinue} { + ECHO; + } +. { + /* This is only needed for \ just before EOF */ + ECHO; + } + +{dolqdelim} { + cur_state->dolqstart = pg_strdup(yytext); + BEGIN(xdolq); + ECHO; + } +{dolqfailed} { + /* throw back all but the initial "$" */ + yyless(1); + ECHO; + } +{dolqdelim} { + if (strcmp(yytext, cur_state->dolqstart) == 0) + { + free(cur_state->dolqstart); + cur_state->dolqstart = NULL; + BEGIN(INITIAL); + } + else + { + /* + * When we fail to match $...$ to dolqstart, transfer + * the $... part to the output, but put back the final + * $ for rescanning. Consider $delim$...$junk$delim$ + */ + yyless(yyleng - 1); + } + ECHO; + } +{dolqinside} { + ECHO; + } +{dolqfailed} { + ECHO; + } +. { + /* This is only needed for $ inside the quoted text */ + ECHO; + } + +{xdstart} { + BEGIN(xd); + ECHO; + } +{xuistart} { + BEGIN(xui); + ECHO; + } +{xdstop} { + BEGIN(INITIAL); + ECHO; + } +{dquote} { + yyless(1); + BEGIN(xuiend); + ECHO; + } +{whitespace} { + ECHO; + } +{other} | +{xustop1} { + yyless(0); + BEGIN(INITIAL); + ECHO; + } +{xustop2} { + BEGIN(INITIAL); + ECHO; + } +{xddouble} { + ECHO; + } +{xdinside} { + ECHO; + } + +{xufailed} { + /* throw back all but the initial u/U */ + yyless(1); + ECHO; + } + +{typecast} { + ECHO; + } + +{dot_dot} { + ECHO; + } + +{colon_equals} { + ECHO; + } + +{equals_greater} { + ECHO; + } + +{less_equals} { + ECHO; + } + +{greater_equals} { + ECHO; + } + +{less_greater} { + ECHO; + } + +{not_equals} { + ECHO; + } + + /* + * These rules are specific to psql --- they implement parenthesis + * counting and detection of command-ending semicolon. These must + * appear before the {self} rule so that they take precedence over it. + */ + +"(" { + cur_state->paren_depth++; + ECHO; + } + +")" { + if (cur_state->paren_depth > 0) + cur_state->paren_depth--; + ECHO; + } + +";" { + ECHO; + if (cur_state->paren_depth == 0) + { + /* Terminate lexing temporarily */ + cur_state->start_state = YY_START; + return LEXRES_SEMI; + } + } + + /* + * psql-specific rules to handle backslash commands and variable + * substitution. We want these before {self}, also. + */ + +"\\"[;:] { + /* Force a semicolon or colon into the query buffer */ + psqlscan_emit(cur_state, yytext + 1, 1); + } + +"\\" { + /* Terminate lexing temporarily */ + cur_state->start_state = YY_START; + return LEXRES_BACKSLASH; + } + +:{variable_char}+ { + /* Possible psql variable substitution */ + char *varname; + char *value; + + varname = psqlscan_extract_substring(cur_state, + yytext + 1, + yyleng - 1); + if (cur_state->callbacks->get_variable) + value = cur_state->callbacks->get_variable(varname, + false, + false); + else + value = NULL; + + if (value) + { + /* It is a variable, check for recursion */ + if (psqlscan_var_is_current_source(cur_state, varname)) + { + /* Recursive expansion --- don't go there */ + cur_state->callbacks->write_error("skipping recursive expansion of variable \"%s\"\n", + varname); + /* Instead copy the string as is */ + ECHO; + } + else + { + /* OK, perform substitution */ + psqlscan_push_new_buffer(cur_state, value, varname); + /* yy_scan_string already made buffer active */ + } + free(value); + } + else + { + /* + * if the variable doesn't exist we'll copy the string + * as is + */ + ECHO; + } + + free(varname); + } + +:'{variable_char}+' { + psqlscan_escape_variable(cur_state, yytext, yyleng, false); + } + +:\"{variable_char}+\" { + psqlscan_escape_variable(cur_state, yytext, yyleng, true); + } + + /* + * These rules just avoid the need for scanner backup if one of the + * two rules above fails to match completely. + */ + +:'{variable_char}* { + /* Throw back everything but the colon */ + yyless(1); + ECHO; + } + +:\"{variable_char}* { + /* Throw back everything but the colon */ + yyless(1); + ECHO; + } + + /* + * Back to backend-compatible rules. + */ + +{self} { + ECHO; + } + +{operator} { + /* + * Check for embedded slash-star or dash-dash; those + * are comment starts, so operator must stop there. + * Note that slash-star or dash-dash at the first + * character will match a prior rule, not this one. + */ + int nchars = yyleng; + char *slashstar = strstr(yytext, "/*"); + char *dashdash = strstr(yytext, "--"); + + if (slashstar && dashdash) + { + /* if both appear, take the first one */ + if (slashstar > dashdash) + slashstar = dashdash; + } + else if (!slashstar) + slashstar = dashdash; + if (slashstar) + nchars = slashstar - yytext; + + /* + * For SQL compatibility, '+' and '-' cannot be the + * last char of a multi-char operator unless the operator + * contains chars that are not in SQL operators. + * The idea is to lex '=-' as two operators, but not + * to forbid operator names like '?-' that could not be + * sequences of SQL operators. + */ + while (nchars > 1 && + (yytext[nchars - 1] == '+' || + yytext[nchars - 1] == '-')) + { + int ic; + + for (ic = nchars - 2; ic >= 0; ic--) + { + if (strchr("~!@#^&|`?%", yytext[ic])) + break; + } + if (ic >= 0) + break; /* found a char that makes it OK */ + nchars--; /* else remove the +/-, and check again */ + } + + if (nchars < yyleng) + { + /* Strip the unwanted chars from the token */ + yyless(nchars); + } + ECHO; + } + +{param} { + ECHO; + } + +{integer} { + ECHO; + } +{decimal} { + ECHO; + } +{decimalfail} { + /* throw back the .., and treat as integer */ + yyless(yyleng - 2); + ECHO; + } +{real} { + ECHO; + } +{realfail1} { + /* + * throw back the [Ee], and treat as {decimal}. Note + * that it is possible the input is actually {integer}, + * but since this case will almost certainly lead to a + * syntax error anyway, we don't bother to distinguish. + */ + yyless(yyleng - 1); + ECHO; + } +{realfail2} { + /* throw back the [Ee][+-], and proceed as above */ + yyless(yyleng - 2); + ECHO; + } + + +{identifier} { + ECHO; + } + +{other} { + ECHO; + } + +<> { + if (cur_state->buffer_stack == NULL) + { + cur_state->start_state = YY_START; + return LEXRES_EOL; /* end of input reached */ + } + + /* + * We were expanding a variable, so pop the inclusion + * stack and keep lexing + */ + psqlscan_pop_buffer_stack(cur_state); + psqlscan_select_top_buffer(cur_state); + } + +%% + +/* + * Create a lexer working state struct. + * + * callbacks is a struct of function pointers that encapsulate some + * behavior we need from the surrounding program. This struct must + * remain valid for the lifespan of the PsqlScanState. + */ +PsqlScanState +psql_scan_create(const PsqlScanCallbacks *callbacks) +{ + PsqlScanState state; + + state = (PsqlScanStateData *) pg_malloc0(sizeof(PsqlScanStateData)); + + state->callbacks = callbacks; + + yylex_init(&state->scanner); + + yyset_extra(state, state->scanner); + + psql_scan_reset(state); + + return state; +} + +/* + * Destroy a lexer working state struct, releasing all resources. + */ +void +psql_scan_destroy(PsqlScanState state) +{ + psql_scan_finish(state); + + psql_scan_reset(state); + + yylex_destroy(state->scanner); + + free(state); +} + +/* + * Set up to perform lexing of the given input line. + * + * The text at *line, extending for line_len bytes, will be scanned by + * subsequent calls to the psql_scan routines. psql_scan_finish should + * be called when scanning is complete. Note that the lexer retains + * a pointer to the storage at *line --- this string must not be altered + * or freed until after psql_scan_finish is called. + * + * encoding is the libpq identifier for the character encoding in use, + * and std_strings says whether standard_conforming_strings is on. + */ +void +psql_scan_setup(PsqlScanState state, + const char *line, int line_len, + int encoding, bool std_strings) +{ + /* Mustn't be scanning already */ + Assert(state->scanbufhandle == NULL); + Assert(state->buffer_stack == NULL); + + /* Do we need to hack the character set encoding? */ + state->encoding = encoding; + state->safe_encoding = pg_valid_server_encoding_id(encoding); + + /* Save standard-strings flag as well */ + state->std_strings = std_strings; + + /* Set up flex input buffer with appropriate translation and padding */ + state->scanbufhandle = psqlscan_prepare_buffer(state, line, line_len, + &state->scanbuf); + state->scanline = line; + + /* Set lookaside data in case we have to map unsafe encoding */ + state->curline = state->scanbuf; + state->refline = state->scanline; +} + +/* + * Do lexical analysis of SQL command text. + * + * The text previously passed to psql_scan_setup is scanned, and appended + * (possibly with transformation) to query_buf. + * + * The return value indicates the condition that stopped scanning: + * + * PSCAN_SEMICOLON: found a command-ending semicolon. (The semicolon is + * transferred to query_buf.) The command accumulated in query_buf should + * be executed, then clear query_buf and call again to scan the remainder + * of the line. + * + * PSCAN_BACKSLASH: found a backslash that starts a special command. + * Any previous data on the line has been transferred to query_buf. + * The caller will typically next apply a separate flex lexer to scan + * the special command. + * + * PSCAN_INCOMPLETE: the end of the line was reached, but we have an + * incomplete SQL command. *prompt is set to the appropriate prompt type. + * + * PSCAN_EOL: the end of the line was reached, and there is no lexical + * reason to consider the command incomplete. The caller may or may not + * choose to send it. *prompt is set to the appropriate prompt type if + * the caller chooses to collect more input. + * + * In the PSCAN_INCOMPLETE and PSCAN_EOL cases, psql_scan_finish() should + * be called next, then the cycle may be repeated with a fresh input line. + * + * In all cases, *prompt is set to an appropriate prompt type code for the + * next line-input operation. + */ +PsqlScanResult +psql_scan(PsqlScanState state, + PQExpBuffer query_buf, + promptStatus_t *prompt) +{ + PsqlScanResult result; + int lexresult; + + /* Must be scanning already */ + Assert(state->scanbufhandle != NULL); + + /* Set current output target */ + state->output_buf = query_buf; + + /* Set input source */ + if (state->buffer_stack != NULL) + yy_switch_to_buffer(state->buffer_stack->buf, state->scanner); + else + yy_switch_to_buffer(state->scanbufhandle, state->scanner); + + /* And lex. */ + lexresult = yylex(NULL, state->scanner); + + /* + * Check termination state and return appropriate result info. + */ + switch (lexresult) + { + case LEXRES_EOL: /* end of input */ + switch (state->start_state) + { + case INITIAL: + case xuiend: /* we treat these like INITIAL */ + case xusend: + if (state->paren_depth > 0) + { + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_PAREN; + } + else if (query_buf->len > 0) + { + result = PSCAN_EOL; + *prompt = PROMPT_CONTINUE; + } + else + { + /* never bother to send an empty buffer */ + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_READY; + } + break; + case xb: + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_SINGLEQUOTE; + break; + case xc: + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_COMMENT; + break; + case xd: + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_DOUBLEQUOTE; + break; + case xh: + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_SINGLEQUOTE; + break; + case xe: + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_SINGLEQUOTE; + break; + case xq: + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_SINGLEQUOTE; + break; + case xdolq: + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_DOLLARQUOTE; + break; + case xui: + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_DOUBLEQUOTE; + break; + case xus: + result = PSCAN_INCOMPLETE; + *prompt = PROMPT_SINGLEQUOTE; + break; + default: + /* can't get here */ + fprintf(stderr, "invalid YY_START\n"); + exit(1); + } + break; + case LEXRES_SEMI: /* semicolon */ + result = PSCAN_SEMICOLON; + *prompt = PROMPT_READY; + break; + case LEXRES_BACKSLASH: /* backslash */ + result = PSCAN_BACKSLASH; + *prompt = PROMPT_READY; + break; + default: + /* can't get here */ + fprintf(stderr, "invalid yylex result\n"); + exit(1); + } + + return result; +} + +/* + * Clean up after scanning a string. This flushes any unread input and + * releases resources (but not the PsqlScanState itself). Note however + * that this does not reset the lexer scan state; that can be done by + * psql_scan_reset(), which is an orthogonal operation. + * + * It is legal to call this when not scanning anything (makes it easier + * to deal with error recovery). + */ +void +psql_scan_finish(PsqlScanState state) +{ + /* Drop any incomplete variable expansions. */ + while (state->buffer_stack != NULL) + psqlscan_pop_buffer_stack(state); + + /* Done with the outer scan buffer, too */ + if (state->scanbufhandle) + yy_delete_buffer(state->scanbufhandle, state->scanner); + state->scanbufhandle = NULL; + if (state->scanbuf) + free(state->scanbuf); + state->scanbuf = NULL; +} + +/* + * Reset lexer scanning state to start conditions. This is appropriate + * for executing \r psql commands (or any other time that we discard the + * prior contents of query_buf). It is not, however, necessary to do this + * when we execute and clear the buffer after getting a PSCAN_SEMICOLON or + * PSCAN_EOL scan result, because the scan state must be INITIAL when those + * conditions are returned. + * + * Note that this is unrelated to flushing unread input; that task is + * done by psql_scan_finish(). + */ +void +psql_scan_reset(PsqlScanState state) +{ + state->start_state = INITIAL; + state->paren_depth = 0; + state->xcdepth = 0; /* not really necessary */ + if (state->dolqstart) + free(state->dolqstart); + state->dolqstart = NULL; +} + +/* + * Reselect this lexer (psqlscan.l) after using another one. + * + * Currently and for foreseeable uses, it's sufficient to reset to INITIAL + * state, because we'd never switch to another lexer in a different state. + * However, we don't want to reset e.g. paren_depth, so this can't be + * the same as psql_scan_reset(). + * + * Note: psql setjmp error recovery just calls psql_scan_reset(), so that + * must be a superset of this. + * + * Note: it seems likely that other lexers could just assign INITIAL for + * themselves, since that probably has the value zero in every flex-generated + * lexer. But let's not assume that. + */ +void +psql_scan_reselect_sql_lexer(PsqlScanState state) +{ + state->start_state = INITIAL; +} + +/* + * Return true if lexer is currently in an "inside quotes" state. + * + * This is pretty grotty but is needed to preserve the old behavior + * that mainloop.c drops blank lines not inside quotes without even + * echoing them. + */ +bool +psql_scan_in_quote(PsqlScanState state) +{ + return state->start_state != INITIAL; +} + +/* + * Push the given string onto the stack of stuff to scan. + * + * NOTE SIDE EFFECT: the new buffer is made the active flex input buffer. + */ +void +psqlscan_push_new_buffer(PsqlScanState state, const char *newstr, + const char *varname) +{ + StackElem *stackelem; + + stackelem = (StackElem *) pg_malloc(sizeof(StackElem)); + + /* + * In current usage, the passed varname points at the current flex input + * buffer; we must copy it before calling psqlscan_prepare_buffer() + * because that will change the buffer state. + */ + stackelem->varname = varname ? pg_strdup(varname) : NULL; + + stackelem->buf = psqlscan_prepare_buffer(state, newstr, strlen(newstr), + &stackelem->bufstring); + state->curline = stackelem->bufstring; + if (state->safe_encoding) + { + stackelem->origstring = NULL; + state->refline = stackelem->bufstring; + } + else + { + stackelem->origstring = pg_strdup(newstr); + state->refline = stackelem->origstring; + } + stackelem->next = state->buffer_stack; + state->buffer_stack = stackelem; +} + +/* + * Pop the topmost buffer stack item (there must be one!) + * + * NB: after this, the flex input state is unspecified; caller must + * switch to an appropriate buffer to continue lexing. + * See psqlscan_select_top_buffer(). + */ +void +psqlscan_pop_buffer_stack(PsqlScanState state) +{ + StackElem *stackelem = state->buffer_stack; + + state->buffer_stack = stackelem->next; + yy_delete_buffer(stackelem->buf, state->scanner); + free(stackelem->bufstring); + if (stackelem->origstring) + free(stackelem->origstring); + if (stackelem->varname) + free(stackelem->varname); + free(stackelem); +} + +/* + * Select the topmost surviving buffer as the active input. + */ +void +psqlscan_select_top_buffer(PsqlScanState state) +{ + StackElem *stackelem = state->buffer_stack; + + if (stackelem != NULL) + { + yy_switch_to_buffer(stackelem->buf, state->scanner); + state->curline = stackelem->bufstring; + state->refline = stackelem->origstring ? stackelem->origstring : stackelem->bufstring; + } + else + { + yy_switch_to_buffer(state->scanbufhandle, state->scanner); + state->curline = state->scanbuf; + state->refline = state->scanline; + } +} + +/* + * Check if specified variable name is the source for any string + * currently being scanned + */ +bool +psqlscan_var_is_current_source(PsqlScanState state, const char *varname) +{ + StackElem *stackelem; + + for (stackelem = state->buffer_stack; + stackelem != NULL; + stackelem = stackelem->next) + { + if (stackelem->varname && strcmp(stackelem->varname, varname) == 0) + return true; + } + return false; +} + +/* + * Set up a flex input buffer to scan the given data. We always make a + * copy of the data. If working in an unsafe encoding, the copy has + * multibyte sequences replaced by FFs to avoid fooling the lexer rules. + * + * NOTE SIDE EFFECT: the new buffer is made the active flex input buffer. + */ +YY_BUFFER_STATE +psqlscan_prepare_buffer(PsqlScanState state, const char *txt, int len, + char **txtcopy) +{ + char *newtxt; + + /* Flex wants two \0 characters after the actual data */ + newtxt = pg_malloc(len + 2); + *txtcopy = newtxt; + newtxt[len] = newtxt[len + 1] = YY_END_OF_BUFFER_CHAR; + + if (state->safe_encoding) + memcpy(newtxt, txt, len); + else + { + /* Gotta do it the hard way */ + int i = 0; + + while (i < len) + { + int thislen = PQmblen(txt + i, state->encoding); + + /* first byte should always be okay... */ + newtxt[i] = txt[i]; + i++; + while (--thislen > 0 && i < len) + newtxt[i++] = (char) 0xFF; + } + } + + return yy_scan_buffer(newtxt, len + 2, state->scanner); +} + +/* + * psqlscan_emit() --- body for ECHO macro + * + * NB: this must be used for ALL and ONLY the text copied from the flex + * input data. If you pass it something that is not part of the yytext + * string, you are making a mistake. Internally generated text can be + * appended directly to state->output_buf. + */ +void +psqlscan_emit(PsqlScanState state, const char *txt, int len) +{ + PQExpBuffer output_buf = state->output_buf; + + if (state->safe_encoding) + appendBinaryPQExpBuffer(output_buf, txt, len); + else + { + /* Gotta do it the hard way */ + const char *reference = state->refline; + int i; + + reference += (txt - state->curline); + + for (i = 0; i < len; i++) + { + char ch = txt[i]; + + if (ch == (char) 0xFF) + ch = reference[i]; + appendPQExpBufferChar(output_buf, ch); + } + } +} + +/* + * psqlscan_extract_substring --- fetch value of (part of) the current token + * + * This is like psqlscan_emit(), except that the data is returned as a + * malloc'd string rather than being pushed directly to state->output_buf. + */ +char * +psqlscan_extract_substring(PsqlScanState state, const char *txt, int len) +{ + char *result = (char *) pg_malloc(len + 1); + + if (state->safe_encoding) + memcpy(result, txt, len); + else + { + /* Gotta do it the hard way */ + const char *reference = state->refline; + int i; + + reference += (txt - state->curline); + + for (i = 0; i < len; i++) + { + char ch = txt[i]; + + if (ch == (char) 0xFF) + ch = reference[i]; + result[i] = ch; + } + } + result[len] = '\0'; + return result; +} + +/* + * psqlscan_escape_variable --- process :'VARIABLE' or :"VARIABLE" + * + * If the variable name is found, escape its value using the appropriate + * quoting method and emit the value to output_buf. (Since the result is + * surely quoted, there is never any reason to rescan it.) If we don't + * find the variable or escaping fails, emit the token as-is. + */ +void +psqlscan_escape_variable(PsqlScanState state, const char *txt, int len, + bool as_ident) +{ + char *varname; + char *value; + + /* Variable lookup. */ + varname = psqlscan_extract_substring(state, txt + 2, len - 3); + if (state->callbacks->get_variable) + value = state->callbacks->get_variable(varname, true, as_ident); + else + value = NULL; + free(varname); + + if (value) + { + /* Emit the suitably-escaped value */ + appendPQExpBufferStr(state->output_buf, value); + free(value); + } + else + { + /* Emit original token as-is */ + psqlscan_emit(state, txt, len); + } +} diff --git a/src/include/fe_utils/psqlscan.h b/src/include/fe_utils/psqlscan.h new file mode 100644 index 00000000000..1f10ecc2d44 --- /dev/null +++ b/src/include/fe_utils/psqlscan.h @@ -0,0 +1,85 @@ +/*------------------------------------------------------------------------- + * + * psqlscan.h + * lexical scanner for SQL commands + * + * This lexer used to be part of psql, and that heritage is reflected in + * the file name as well as function and typedef names, though it can now + * be used by other frontend programs as well. It's also possible to extend + * this lexer with a compatible add-on lexer to handle program-specific + * backslash commands. + * + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/fe_utils/psqlscan.h + * + *------------------------------------------------------------------------- + */ +#ifndef PSQLSCAN_H +#define PSQLSCAN_H + +#include "pqexpbuffer.h" + + +/* Abstract type for lexer's internal state */ +typedef struct PsqlScanStateData *PsqlScanState; + +/* Termination states for psql_scan() */ +typedef enum +{ + PSCAN_SEMICOLON, /* found command-ending semicolon */ + PSCAN_BACKSLASH, /* found backslash command */ + PSCAN_INCOMPLETE, /* end of line, SQL statement incomplete */ + PSCAN_EOL /* end of line, SQL possibly complete */ +} PsqlScanResult; + +/* Prompt type returned by psql_scan() */ +typedef enum _promptStatus +{ + PROMPT_READY, + PROMPT_CONTINUE, + PROMPT_COMMENT, + PROMPT_SINGLEQUOTE, + PROMPT_DOUBLEQUOTE, + PROMPT_DOLLARQUOTE, + PROMPT_PAREN, + PROMPT_COPY +} promptStatus_t; + +/* Callback functions to be used by the lexer */ +typedef struct PsqlScanCallbacks +{ + /* Fetch value of a variable, as a pfree'able string; NULL if unknown */ + /* This pointer can be NULL if no variable substitution is wanted */ + char *(*get_variable) (const char *varname, bool escape, bool as_ident); + /* Print an error message someplace appropriate */ + /* (very old gcc versions don't support attributes on function pointers) */ +#if defined(__GNUC__) && __GNUC__ < 4 + void (*write_error) (const char *fmt,...); +#else + void (*write_error) (const char *fmt,...) pg_attribute_printf(1, 2); +#endif +} PsqlScanCallbacks; + + +extern PsqlScanState psql_scan_create(const PsqlScanCallbacks *callbacks); +extern void psql_scan_destroy(PsqlScanState state); + +extern void psql_scan_setup(PsqlScanState state, + const char *line, int line_len, + int encoding, bool std_strings); +extern void psql_scan_finish(PsqlScanState state); + +extern PsqlScanResult psql_scan(PsqlScanState state, + PQExpBuffer query_buf, + promptStatus_t *prompt); + +extern void psql_scan_reset(PsqlScanState state); + +extern void psql_scan_reselect_sql_lexer(PsqlScanState state); + +extern bool psql_scan_in_quote(PsqlScanState state); + +#endif /* PSQLSCAN_H */ diff --git a/src/include/fe_utils/psqlscan_int.h b/src/include/fe_utils/psqlscan_int.h new file mode 100644 index 00000000000..a52929d5abf --- /dev/null +++ b/src/include/fe_utils/psqlscan_int.h @@ -0,0 +1,144 @@ +/*------------------------------------------------------------------------- + * + * psqlscan_int.h + * lexical scanner internal declarations + * + * This file declares the PsqlScanStateData structure used by psqlscan.l + * and shared by other lexers compatible with it, such as psqlscanslash.l. + * + * One difficult aspect of this code is that we need to work in multibyte + * encodings that are not ASCII-safe. A "safe" encoding is one in which each + * byte of a multibyte character has the high bit set (it's >= 0x80). Since + * all our lexing rules treat all high-bit-set characters alike, we don't + * really need to care whether such a byte is part of a sequence or not. + * In an "unsafe" encoding, we still expect the first byte of a multibyte + * sequence to be >= 0x80, but later bytes might not be. If we scan such + * a sequence as-is, the lexing rules could easily be fooled into matching + * such bytes to ordinary ASCII characters. Our solution for this is to + * substitute 0xFF for each non-first byte within the data presented to flex. + * The flex rules will then pass the FF's through unmolested. The + * psqlscan_emit() subroutine is responsible for looking back to the original + * string and replacing FF's with the corresponding original bytes. + * + * Another interesting thing we do here is scan different parts of the same + * input with physically separate flex lexers (ie, lexers written in separate + * .l files). We can get away with this because the only part of the + * persistent state of a flex lexer that depends on its parsing rule tables + * is the start state number, which is easy enough to manage --- usually, + * in fact, we just need to set it to INITIAL when changing lexers. But to + * make that work at all, we must use re-entrant lexers, so that all the + * relevant state is in the yyscanner_t attached to the PsqlScanState; + * if we were using lexers with separate static state we would soon end up + * with dangling buffer pointers in one or the other. Also note that this + * is unlikely to work very nicely if the lexers aren't all built with the + * same flex version, or if they don't use the same flex options. + * + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/fe_utils/psqlscan_int.h + * + *------------------------------------------------------------------------- + */ +#ifndef PSQLSCAN_INT_H +#define PSQLSCAN_INT_H + +#include "fe_utils/psqlscan.h" + +/* + * These are just to allow this file to be compilable standalone for header + * validity checking; in actual use, this file should always be included + * from the body of a flex file, where these symbols are already defined. + */ +#ifndef YY_TYPEDEF_YY_BUFFER_STATE +#define YY_TYPEDEF_YY_BUFFER_STATE +typedef struct yy_buffer_state *YY_BUFFER_STATE; +#endif +#ifndef YY_TYPEDEF_YY_SCANNER_T +#define YY_TYPEDEF_YY_SCANNER_T +typedef void *yyscan_t; +#endif + +/* + * We use a stack of flex buffers to handle substitution of psql variables. + * Each stacked buffer contains the as-yet-unread text from one psql variable. + * When we pop the stack all the way, we resume reading from the outer buffer + * identified by scanbufhandle. + */ +typedef struct StackElem +{ + YY_BUFFER_STATE buf; /* flex input control structure */ + char *bufstring; /* data actually being scanned by flex */ + char *origstring; /* copy of original data, if needed */ + char *varname; /* name of variable providing data, or NULL */ + struct StackElem *next; +} StackElem; + +/* + * All working state of the lexer must be stored in PsqlScanStateData + * between calls. This allows us to have multiple open lexer operations, + * which is needed for nested include files. The lexer itself is not + * recursive, but it must be re-entrant. + */ +typedef struct PsqlScanStateData +{ + yyscan_t scanner; /* Flex's state for this PsqlScanState */ + + PQExpBuffer output_buf; /* current output buffer */ + + StackElem *buffer_stack; /* stack of variable expansion buffers */ + + /* + * These variables always refer to the outer buffer, never to any stacked + * variable-expansion buffer. + */ + YY_BUFFER_STATE scanbufhandle; + char *scanbuf; /* start of outer-level input buffer */ + const char *scanline; /* current input line at outer level */ + + /* safe_encoding, curline, refline are used by emit() to replace FFs */ + int encoding; /* encoding being used now */ + bool safe_encoding; /* is current encoding "safe"? */ + bool std_strings; /* are string literals standard? */ + const char *curline; /* actual flex input string for cur buf */ + const char *refline; /* original data for cur buffer */ + + /* + * All this state lives across successive input lines, until explicitly + * reset by psql_scan_reset. start_state is adopted by yylex() on entry, + * and updated with its finishing state on exit. + */ + int start_state; /* yylex's starting/finishing state */ + int paren_depth; /* depth of nesting in parentheses */ + int xcdepth; /* depth of nesting in slash-star comments */ + char *dolqstart; /* current $foo$ quote start string */ + + /* + * Callback functions provided by the program making use of the lexer. + */ + const PsqlScanCallbacks *callbacks; +} PsqlScanStateData; + + +/* + * Functions exported by psqlscan.l, but only meant for use within + * compatible lexers. + */ +extern void psqlscan_push_new_buffer(PsqlScanState state, + const char *newstr, const char *varname); +extern void psqlscan_pop_buffer_stack(PsqlScanState state); +extern void psqlscan_select_top_buffer(PsqlScanState state); +extern bool psqlscan_var_is_current_source(PsqlScanState state, + const char *varname); +extern YY_BUFFER_STATE psqlscan_prepare_buffer(PsqlScanState state, + const char *txt, int len, + char **txtcopy); +extern void psqlscan_emit(PsqlScanState state, const char *txt, int len); +extern char *psqlscan_extract_substring(PsqlScanState state, + const char *txt, int len); +extern void psqlscan_escape_variable(PsqlScanState state, + const char *txt, int len, + bool as_ident); + +#endif /* PSQLSCAN_INT_H */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 1cd8452f49e..ebc2da8e844 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -63,16 +63,14 @@ my $frontend_extralibs = { 'psql' => ['ws2_32.lib'] }; my $frontend_extraincludes = { 'initdb' => ['src/timezone'], - 'psql' => [ 'src/backend' ], - 'pgbench' => [ 'src/bin/psql' ] }; + 'psql' => [ 'src/backend' ] }; my $frontend_extrasource = { - 'psql' => ['src/bin/psql/psqlscan.l', 'src/bin/psql/psqlscanslash.l'], + 'psql' => [ 'src/bin/psql/psqlscanslash.l' ], 'pgbench' => - [ 'src/bin/pgbench/exprscan.l', 'src/bin/pgbench/exprparse.y', - 'src/bin/psql/psqlscan.l' ] }; + [ 'src/bin/pgbench/exprscan.l', 'src/bin/pgbench/exprparse.y' ] }; my @frontend_excludes = ( 'pgevent', 'pg_basebackup', 'pg_rewind', 'pg_dump', - 'pg_xlogdump', 'scripts', 'pgbench'); + 'pg_xlogdump', 'scripts'); sub mkvcbuild { @@ -120,7 +118,7 @@ sub mkvcbuild our @pgcommonbkndfiles = @pgcommonallfiles; our @pgfeutilsfiles = qw( - mbprint.c print.c simple_list.c string_utils.c); + mbprint.c print.c psqlscan.l psqlscan.c simple_list.c string_utils.c); $libpgport = $solution->AddProject('libpgport', 'lib', 'misc'); $libpgport->AddDefine('FRONTEND'); @@ -659,11 +657,6 @@ sub mkvcbuild } $pg_xlogdump->AddFile('src/backend/access/transam/xlogreader.c'); - # fix up pgbench once it's been set up - # we're borrowing psqlscan.c from psql, so grab it from the correct place - my $pgbench = AddSimpleFrontend('pgbench'); - $pgbench->ReplaceFile('src/bin/pgbench/psqlscan.c', 'src/bin/psql/psqlscan.c'); - $solution->Save(); return $solution->{vcver}; } diff --git a/src/tools/msvc/clean.bat b/src/tools/msvc/clean.bat index ecf92700f28..469b8a24b29 100755 --- a/src/tools/msvc/clean.bat +++ b/src/tools/msvc/clean.bat @@ -75,7 +75,7 @@ if exist src\pl\plperl\spi.c del /q src\pl\plperl\spi.c if %DIST%==1 if exist src\pl\plpgsql\src\pl_gram.c del /q src\pl\plpgsql\src\pl_gram.c if %DIST%==1 if exist src\pl\plpgsql\src\pl_gram.h del /q src\pl\plpgsql\src\pl_gram.h -if %DIST%==1 if exist src\bin\psql\psqlscan.c del /q src\bin\psql\psqlscan.c +if %DIST%==1 if exist src\fe_utils\psqlscan.c del /q src\fe_utils\psqlscan.c if %DIST%==1 if exist src\bin\psql\psqlscanslash.c del /q src\bin\psql\psqlscanslash.c if %DIST%==1 if exist contrib\cube\cubescan.c del /q contrib\cube\cubescan.c -- cgit v1.2.3