Prevent mis-encoding of "trailing junk after numeric literal" errors.

Since commit 2549f0661, we reject an identifier immediately following
a numeric literal (without separating whitespace), because that risks
ambiguity with hex/octal/binary integers.  However, that patch used
token patterns like "{integer}{ident_start}", which is problematic
because {ident_start} matches only a single byte.  If the first
character after the integer is a multibyte character, this ends up
with flex reporting an error message that includes a partial multibyte
character.  That can cause assorted bad-encoding problems downstream,
both in the report to the client and in the postmaster log file.

To fix, use {identifier} not {ident_start} in the "junk" token
patterns, so that they will match complete multibyte characters.
This seems generally better user experience quite aside from the
encoding problem: for "123abc" the error message will now say that
the error appeared at or near "123abc" instead of "123a".

While at it, add some commentary about why these patterns exist
and how they work.

Report and patch by Karina Litskevich; review by Pavel Borisov.
Back-patch to v15 where the problem came in.

Discussion: https://postgr.es/m/CACiT8iZ_diop=0zJ7zuY3BXegJpkKK1Av-PU7xh0EDYHsa5+=g@mail.gmail.com

diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
index 882e081aae2eb5739e57a48bf4d0ff6660b6a4a3..1638ae834cf7a4fc8999977e40e4f7c6e2a27696 100644 (file)
--- a/src/backend/parser/scan.l
+++ b/src/backend/parser/scan.l
@@ -398,12 +398,12 @@ decimalfail       {digit}+\.\.
 real           ({integer}|{decimal})[Ee][-+]?{digit}+
 realfail       ({integer}|{decimal})[Ee][-+]
 
-integer_junk   {integer}{ident_start}
-decimal_junk   {decimal}{ident_start}
-real_junk      {real}{ident_start}
+integer_junk   {integer}{identifier}
+decimal_junk   {decimal}{identifier}
+real_junk      {real}{identifier}
 
 param          \${integer}
-param_junk     \${integer}{ident_start}
+param_junk     \${integer}{identifier}
 
 other          .
 

diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l
index ae531ec2407793135b85590ae27e0fb5d0040088..6c603f6e7a011bfb386d83b1ab4c9e49f1e4159d 100644 (file)
--- a/src/fe_utils/psqlscan.l
+++ b/src/fe_utils/psqlscan.l
@@ -336,12 +336,12 @@ decimalfail       {digit}+\.\.
 real           ({integer}|{decimal})[Ee][-+]?{digit}+
 realfail       ({integer}|{decimal})[Ee][-+]
 
-integer_junk   {integer}{ident_start}
-decimal_junk   {decimal}{ident_start}
-real_junk      {real}{ident_start}
+integer_junk   {integer}{identifier}
+decimal_junk   {decimal}{identifier}
+real_junk      {real}{identifier}
 
 param          \${integer}
-param_junk     \${integer}{ident_start}
+param_junk     \${integer}{identifier}
 
 /* psql-specific: characters allowed in variable names */
 variable_char  [A-Za-z\200-\377_0-9]

diff --git a/src/interfaces/ecpg/preproc/pgc.l b/src/interfaces/ecpg/preproc/pgc.l
index c06b2d6f51ef13305f94d8afb40648c12b377398..3b16c70bce042f89459a0d6970a1f66e2832d88a 100644 (file)
--- a/src/interfaces/ecpg/preproc/pgc.l
+++ b/src/interfaces/ecpg/preproc/pgc.l
@@ -369,12 +369,12 @@ decimalfail       {digit}+\.\.
 real           ({integer}|{decimal})[Ee][-+]?{digit}+
 realfail       ({integer}|{decimal})[Ee][-+]
 
-integer_junk   {integer}{ident_start}
-decimal_junk   {decimal}{ident_start}
-real_junk      {real}{ident_start}
+integer_junk   {integer}{identifier}
+decimal_junk   {decimal}{identifier}
+real_junk      {real}{identifier}
 
 param          \${integer}
-param_junk     \${integer}{ident_start}
+param_junk     \${integer}{identifier}
 
 /* special characters for other dbms */
 /* we have to react differently in compat mode */

diff --git a/src/test/regress/expected/numerology.out b/src/test/regress/expected/numerology.out
index 77d48434173bb6991d731c89a7a0e190feac2b1c..c369e5ff738c38c58f7230fbe3192237bfc62220 100644 (file)
--- a/src/test/regress/expected/numerology.out
+++ b/src/test/regress/expected/numerology.out
@@ -6,15 +6,15 @@
 -- Trailing junk in numeric literals
 --
 SELECT 123abc;
-ERROR:  trailing junk after numeric literal at or near "123a"
+ERROR:  trailing junk after numeric literal at or near "123abc"
 LINE 1: SELECT 123abc;
                ^
 SELECT 0x0o;
-ERROR:  trailing junk after numeric literal at or near "0x"
+ERROR:  trailing junk after numeric literal at or near "0x0o"
 LINE 1: SELECT 0x0o;
                ^
 SELECT 1_2_3;
-ERROR:  trailing junk after numeric literal at or near "1_"
+ERROR:  trailing junk after numeric literal at or near "1_2_3"
 LINE 1: SELECT 1_2_3;
                ^
 SELECT 0.a;