cube: pure parser and reentrant scanner

Use the flex %option reentrant and the bison option %pure-parser to
make the generated scanner and parser pure, reentrant, and
thread-safe.

Make the generated scanner use palloc() etc. instead of malloc() etc.
Previously, we only used palloc() for the buffer, but flex would still
use malloc() for its internal structures.  As a result, there could be
some small memory leaks in case of uncaught errors.  (We do catch
normal syntax errors as soft errors.)  Now, all the memory is under
palloc() control, so there are no more such issues.

Simplify flex scan buffer management: Instead of constructing the
buffer from pieces and then using yy_scan_buffer(), we can just use
yy_scan_string(), which does the same thing internally.  (Actually, we
use yy_scan_bytes() here because we already have the length.)

The previous code was necessary because we allocated the buffer with
palloc() and the rest of the state was handled by malloc().  But this
is no longer the case; everything is under palloc() now.

(We could even get rid of the yylex_destroy() call and just let the
memory context cleanup handle everything.  But for now, we preserve
the existing behavior.)

Reviewed-by: Heikki Linnakangas <hlinnaka@iki.fi>
Reviewed-by: Andreas Karlsson <andreas@proxel.se>
Discussion: https://www.postgresql.org/message-id/flat/eb6faeac-2a8a-4b69-9189-c33c520e5b7b@eisentraut.org

diff --git a/contrib/cube/cube.c b/contrib/cube/cube.c
index 1fc447511a100a37d92ae260064357efd259de0c..bf8fc489dca378e8eadfd03fc631fdb9f969727a 100644 (file)
--- a/contrib/cube/cube.c
+++ b/contrib/cube/cube.c
@@ -120,13 +120,14 @@ cube_in(PG_FUNCTION_ARGS)
        char       *str = PG_GETARG_CSTRING(0);
        NDBOX      *result;
        Size            scanbuflen;
+       yyscan_t        scanner;
 
-       cube_scanner_init(str, &scanbuflen);
+       cube_scanner_init(str, &scanbuflen, &scanner);
 
-       cube_yyparse(&result, scanbuflen, fcinfo->context);
+       cube_yyparse(&result, scanbuflen, fcinfo->context, scanner);
 
        /* We might as well run this even on failure. */
-       cube_scanner_finish();
+       cube_scanner_finish(scanner);
 
        PG_RETURN_NDBOX_P(result);
 }

diff --git a/contrib/cube/cubedata.h b/contrib/cube/cubedata.h
index 96fa41a04e710bb31380d3caf87ab2ebe1dad437..8bfcc6e99a27d4a37ca7ad3cd0757f7f9c5d7175 100644 (file)
--- a/contrib/cube/cubedata.h
+++ b/contrib/cube/cubedata.h
@@ -59,14 +59,21 @@ typedef struct NDBOX
 #define CubeKNNDistanceEuclid                  17      /* <-> */
 #define CubeKNNDistanceChebyshev               18      /* <=> */
 
+/* for cubescan.l and cubeparse.y */
+/* All grammar constructs return strings */
+#define YYSTYPE char *
+typedef void *yyscan_t;
+
 /* in cubescan.l */
-extern int     cube_yylex(void);
+extern int     cube_yylex(YYSTYPE *yylval_param, yyscan_t yyscanner);
 extern void cube_yyerror(NDBOX **result, Size scanbuflen,
                                                 struct Node *escontext,
+                                                yyscan_t yyscanner,
                                                 const char *message);
-extern void cube_scanner_init(const char *str, Size *scanbuflen);
-extern void cube_scanner_finish(void);
+extern void cube_scanner_init(const char *str, Size *scanbuflen, yyscan_t *yyscannerp);
+extern void cube_scanner_finish(yyscan_t yyscanner);
 
 /* in cubeparse.y */
 extern int     cube_yyparse(NDBOX **result, Size scanbuflen,
-                                                struct Node *escontext);
+                                                struct Node *escontext,
+                                                yyscan_t yyscanner);

diff --git a/contrib/cube/cubeparse.y b/contrib/cube/cubeparse.y
index 52622875cbb883f61fe81d2c198253aa7a2d36dd..a6b7e70630d36a8e47d0df5fa4017bbe124aa404 100644 (file)
--- a/contrib/cube/cubeparse.y
+++ b/contrib/cube/cubeparse.y
@@ -7,19 +7,11 @@
 #include "postgres.h"
 
 #include "cubedata.h"
+#include "cubeparse.h" /* must be after cubedata.h for YYSTYPE and NDBOX */
 #include "nodes/miscnodes.h"
 #include "utils/float.h"
 #include "varatt.h"
 
-/* All grammar constructs return strings */
-#define YYSTYPE char *
-
-#include "cubeparse.h"
-
-/* silence -Wmissing-variable-declarations */
-extern int cube_yychar;
-extern int cube_yynerrs;
-
 /*
  * Bison doesn't allocate anything that needs to live across parser calls,
  * so we can easily have it use palloc instead of malloc.  This prevents
@@ -40,6 +32,9 @@ static bool write_point_as_box(int dim, char *str,
 %parse-param {NDBOX **result}
 %parse-param {Size scanbuflen}
 %parse-param {struct Node *escontext}
+%parse-param {yyscan_t yyscanner}
+%lex-param   {yyscan_t yyscanner}
+%pure-parser
 %expect 0
 %name-prefix="cube_yy"
 
@@ -75,6 +70,8 @@ box: O_BRACKET paren_list COMMA paren_list C_BRACKET
 
                if (!write_box(dim, $2, $4, result, escontext))
                        YYABORT;
+
+               (void) yynerrs; /* suppress compiler warning */
        }
 
        | paren_list COMMA paren_list

diff --git a/contrib/cube/cubescan.l b/contrib/cube/cubescan.l
index a30fbfc3111a25c2493931bc81d9dc7fbf2e7b17..eed324d6e3b7f2be467c169ce0fd8bf4cf94b213 100644 (file)
--- a/contrib/cube/cubescan.l
+++ b/contrib/cube/cubescan.l
@@ -6,13 +6,8 @@
 
 #include "postgres.h"
 
-/*
- * NB: include cubeparse.h only AFTER defining YYSTYPE (to match cubeparse.y)
- * and cubedata.h for NDBOX.
- */
 #include "cubedata.h"
-#define YYSTYPE char *
-#include "cubeparse.h"
+#include "cubeparse.h" /* must be after cubedata.h for YYSTYPE and NDBOX */
 }
 
 %{
@@ -30,18 +25,19 @@ fprintf_to_ereport(const char *fmt, const char *msg)
 {
        ereport(ERROR, (errmsg_internal("%s", msg)));
 }
-
-/* Handles to the buffer that the lexer uses internally */
-static YY_BUFFER_STATE scanbufhandle;
-static char *scanbuf;
 %}
 
+%option reentrant
+%option bison-bridge
 %option 8bit
 %option never-interactive
 %option nodefault
 %option noinput
 %option nounput
 %option noyywrap
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
 %option warn
 %option prefix="cube_yy"
 
@@ -55,14 +51,14 @@ NaN          [nN][aA][nN]
 
 %%
 
-{float}      cube_yylval = yytext; return CUBEFLOAT;
-{infinity}   cube_yylval = yytext; return CUBEFLOAT;
-{NaN}        cube_yylval = yytext; return CUBEFLOAT;
-\[           cube_yylval = "("; return O_BRACKET;
-\]           cube_yylval = ")"; return C_BRACKET;
-\(           cube_yylval = "("; return O_PAREN;
-\)           cube_yylval = ")"; return C_PAREN;
-\,           cube_yylval = ","; return COMMA;
+{float}      *yylval = yytext; return CUBEFLOAT;
+{infinity}   *yylval = yytext; return CUBEFLOAT;
+{NaN}        *yylval = yytext; return CUBEFLOAT;
+\[           *yylval = "("; return O_BRACKET;
+\]           *yylval = ")"; return C_BRACKET;
+\(           *yylval = "("; return O_PAREN;
+\)           *yylval = ")"; return C_PAREN;
+\,           *yylval = ","; return COMMA;
 [ \t\n\r\f\v]+ /* discard spaces */
 .            return yytext[0]; /* alert parser of the garbage */
 
@@ -74,8 +70,11 @@ NaN          [nN][aA][nN]
 void
 cube_yyerror(NDBOX **result, Size scanbuflen,
                         struct Node *escontext,
+                        yyscan_t yyscanner,
                         const char *message)
 {
+       struct yyguts_t * yyg = (struct yyguts_t *) yyscanner;  /* needed for yytext macro */
+
        if (*yytext == YY_END_OF_BUFFER_CHAR)
        {
                errsave(escontext,
@@ -99,26 +98,18 @@ cube_yyerror(NDBOX **result, Size scanbuflen,
  * Called before any actual parsing is done
  */
 void
-cube_scanner_init(const char *str, Size *scanbuflen)
+cube_scanner_init(const char *str, Size *scanbuflen, yyscan_t *yyscannerp)
 {
        Size            slen = strlen(str);
+       yyscan_t        yyscanner;
 
-       /*
-        * Might be left over after ereport()
-        */
-       if (YY_CURRENT_BUFFER)
-               yy_delete_buffer(YY_CURRENT_BUFFER);
+       if (yylex_init(yyscannerp) != 0)
+               elog(ERROR, "yylex_init() failed: %m");
 
-       /*
-        * Make a scan buffer with special termination needed by flex.
-        */
-       *scanbuflen = slen;
-       scanbuf = palloc(slen + 2);
-       memcpy(scanbuf, str, slen);
-       scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
-       scanbufhandle = yy_scan_buffer(scanbuf, slen + 2);
+       yyscanner = *yyscannerp;
 
-       BEGIN(INITIAL);
+       yy_scan_bytes(str, slen, yyscanner);
+       *scanbuflen = slen;
 }
 
 
@@ -126,8 +117,34 @@ cube_scanner_init(const char *str, Size *scanbuflen)
  * Called after parsing is done to clean up after cube_scanner_init()
  */
 void
-cube_scanner_finish(void)
+cube_scanner_finish(yyscan_t yyscanner)
+{
+       yylex_destroy(yyscanner);
+}
+
+/*
+ * Interface functions to make flex use palloc() instead of malloc().
+ * It'd be better to make these static, but flex insists otherwise.
+ */
+
+void *
+yyalloc(yy_size_t size, yyscan_t yyscanner)
+{
+       return palloc(size);
+}
+
+void *
+yyrealloc(void *ptr, yy_size_t size, yyscan_t yyscanner)
+{
+       if (ptr)
+               return repalloc(ptr, size);
+       else
+               return palloc(size);
+}
+
+void
+yyfree(void *ptr, yyscan_t yyscanner)
 {
-       yy_delete_buffer(scanbufhandle);
-       pfree(scanbuf);
+       if (ptr)
+               pfree(ptr);
 }