diff options
| author | Tom Lane | 2016-12-21 20:18:25 +0000 |
|---|---|---|
| committer | Tom Lane | 2016-12-21 20:18:39 +0000 |
| commit | 89fcea1ace40bc025beea2758a80bcd56a319a6f (patch) | |
| tree | 5ebfb4979c8eb3c4052eebae98d26017ec72042e /src/test | |
| parent | 2d1018ca56f5ddaf0bfb5b4d0133283f3e823301 (diff) | |
Fix strange behavior (and possible crashes) in full text phrase search.
In an attempt to simplify the tsquery matching engine, the original
phrase search patch invented rewrite rules that would rearrange a
tsquery so that no AND/OR/NOT operator appeared below a PHRASE operator.
But this approach had numerous problems. The rearrangement step was
missed by ts_rewrite (and perhaps other places), allowing tsqueries
to be created that would cause Assert failures or perhaps crashes at
execution, as reported by Andreas Seltenreich. The rewrite rules
effectively defined semantics for operators underneath PHRASE that were
buggy, or at least unintuitive. And because rewriting was done in
tsqueryin() rather than at execution, the rearrangement was user-visible,
which is not very desirable --- for example, it might cause unexpected
matches or failures to match in ts_rewrite.
As a somewhat independent problem, the behavior of nested PHRASE operators
was only sane for left-deep trees; queries like "x <-> (y <-> z)" did not
behave intuitively at all.
To fix, get rid of the rewrite logic altogether, and instead teach the
tsquery execution engine to manage AND/OR/NOT below a PHRASE operator
by explicitly computing the match location(s) and match widths for these
operators.
This requires introducing some additional fields into the publicly visible
ExecPhraseData struct; but since there's no way for third-party code to
pass such a struct to TS_phrase_execute, it shouldn't create an ABI problem
as long as we don't move the offsets of the existing fields.
Another related problem was that index searches supposed that "!x <-> y"
could be lossily approximated as "!x & y", which isn't correct because
the latter will reject, say, "x q y" which the query itself accepts.
This required some tweaking in TS_execute_ternary along with the main
tsquery engine.
Back-patch to 9.6 where phrase operators were introduced. While this
could be argued to change behavior more than we'd like in a stable branch,
we have to do something about the crash hazards and index-vs-seqscan
inconsistency, and it doesn't seem desirable to let the unintuitive
behaviors induced by the rewriting implementation stand as precedent.
Discussion: https://postgr.es/m/28215.1481999808@sss.pgh.pa.us
Discussion: https://postgr.es/m/26706.1482087250@sss.pgh.pa.us
Diffstat (limited to 'src/test')
| -rw-r--r-- | src/test/regress/expected/tsdicts.out | 12 | ||||
| -rw-r--r-- | src/test/regress/expected/tsearch.out | 40 | ||||
| -rw-r--r-- | src/test/regress/expected/tstypes.out | 298 | ||||
| -rw-r--r-- | src/test/regress/sql/tsearch.sql | 6 | ||||
| -rw-r--r-- | src/test/regress/sql/tstypes.sql | 56 |
5 files changed, 220 insertions, 192 deletions
diff --git a/src/test/regress/expected/tsdicts.out b/src/test/regress/expected/tsdicts.out index c55591a6787..8ed64d3c68e 100644 --- a/src/test/regress/expected/tsdicts.out +++ b/src/test/regress/expected/tsdicts.out @@ -470,15 +470,15 @@ SELECT to_tsquery('hunspell_tst', 'footballyklubber:b & rebookings:A & sky'); (1 row) SELECT to_tsquery('hunspell_tst', 'footballyklubber:b <-> sky'); - to_tsquery ------------------------------------------------------------------ - 'foot':B <-> 'sky' & 'ball':B <-> 'sky' & 'klubber':B <-> 'sky' + to_tsquery +------------------------------------------------- + ( 'foot':B & 'ball':B & 'klubber':B ) <-> 'sky' (1 row) SELECT phraseto_tsquery('hunspell_tst', 'footballyklubber sky'); - phraseto_tsquery ------------------------------------------------------------ - 'foot' <-> 'sky' & 'ball' <-> 'sky' & 'klubber' <-> 'sky' + phraseto_tsquery +------------------------------------------- + ( 'foot' & 'ball' & 'klubber' ) <-> 'sky' (1 row) -- Test ispell dictionary with hunspell affix with FLAG long in configuration diff --git a/src/test/regress/expected/tsearch.out b/src/test/regress/expected/tsearch.out index cf3beb35a99..0681d43358b 100644 --- a/src/test/regress/expected/tsearch.out +++ b/src/test/regress/expected/tsearch.out @@ -556,15 +556,15 @@ SELECT plainto_tsquery('english', 'foo bar') && 'asd | fg'; -- Check stop word deletion, a and s are stop-words SELECT to_tsquery('english', '!(a & !b) & c'); - to_tsquery ------------- - 'b' & 'c' + to_tsquery +------------- + !!'b' & 'c' (1 row) SELECT to_tsquery('english', '!(a & !b)'); to_tsquery ------------ - 'b' + !!'b' (1 row) SELECT to_tsquery('english', '(1 <-> 2) <-> a'); @@ -1240,15 +1240,15 @@ SELECT ts_rewrite('1 & (2 <2> 3)', 'SELECT keyword, sample FROM test_tsquery'::t (1 row) SELECT ts_rewrite('5 <-> (1 & (2 <-> 3))', 'SELECT keyword, sample FROM test_tsquery'::text ); - ts_rewrite ---------------------------------------- - '5' <-> '1' & '5' <-> ( '2' <-> '3' ) + ts_rewrite +------------------------- + '5' <-> ( '2' <-> '4' ) (1 row) SELECT ts_rewrite('5 <-> (6 | 8)', 'SELECT keyword, sample FROM test_tsquery'::text ); - ts_rewrite ---------------------------- - '5' <-> '7' | '5' <-> '8' + ts_rewrite +----------------------- + '5' <-> ( '6' | '8' ) (1 row) -- Check empty substitution @@ -1386,6 +1386,26 @@ SELECT ts_rewrite( query, 'SELECT keyword, sample FROM test_tsquery' ) FROM to_t 'citi' & 'foo' & ( 'bar' | 'qq' ) & ( 'nyc' | 'big' & 'appl' | 'new' & 'york' ) (1 row) +SELECT ts_rewrite(tsquery_phrase('foo', 'foo'), 'foo', 'bar | baz'); + ts_rewrite +----------------------------------------- + ( 'bar' | 'baz' ) <-> ( 'bar' | 'baz' ) +(1 row) + +SELECT to_tsvector('foo bar') @@ + ts_rewrite(tsquery_phrase('foo', 'foo'), 'foo', 'bar | baz'); + ?column? +---------- + f +(1 row) + +SELECT to_tsvector('bar baz') @@ + ts_rewrite(tsquery_phrase('foo', 'foo'), 'foo', 'bar | baz'); + ?column? +---------- + t +(1 row) + RESET enable_seqscan; --test GUC SET default_text_search_config=simple; diff --git a/src/test/regress/expected/tstypes.out b/src/test/regress/expected/tstypes.out index 8d9290cbac1..dcce82fdc4f 100644 --- a/src/test/regress/expected/tstypes.out +++ b/src/test/regress/expected/tstypes.out @@ -366,133 +366,6 @@ SELECT '!!a & !!b'::tsquery; !!'a' & !!'b' (1 row) --- phrase transformation -SELECT 'a <-> (b|c)'::tsquery; - tsquery ---------------------------- - 'a' <-> 'b' | 'a' <-> 'c' -(1 row) - -SELECT '(a|b) <-> c'::tsquery; - tsquery ---------------------------- - 'a' <-> 'c' | 'b' <-> 'c' -(1 row) - -SELECT '(a|b) <-> (d|c)'::tsquery; - tsquery -------------------------------------------------------- - 'a' <-> 'd' | 'b' <-> 'd' | 'a' <-> 'c' | 'b' <-> 'c' -(1 row) - -SELECT 'a <-> (b&c)'::tsquery; - tsquery ---------------------------- - 'a' <-> 'b' & 'a' <-> 'c' -(1 row) - -SELECT '(a&b) <-> c'::tsquery; - tsquery ---------------------------- - 'a' <-> 'c' & 'b' <-> 'c' -(1 row) - -SELECT '(a&b) <-> (d&c)'::tsquery; - tsquery -------------------------------------------------------- - 'a' <-> 'd' & 'b' <-> 'd' & 'a' <-> 'c' & 'b' <-> 'c' -(1 row) - -SELECT 'a <-> !b'::tsquery; - tsquery ------------------------- - 'a' & !( 'a' <-> 'b' ) -(1 row) - -SELECT '!a <-> b'::tsquery; - tsquery ------------------------- - !( 'a' <-> 'b' ) & 'b' -(1 row) - -SELECT '!a <-> !b'::tsquery; - tsquery ------------------------------------- - !'a' & !( !( 'a' <-> 'b' ) & 'b' ) -(1 row) - -SELECT 'a <-> !(b&c)'::tsquery; - tsquery --------------------------------------- - 'a' & !( 'a' <-> 'b' & 'a' <-> 'c' ) -(1 row) - -SELECT 'a <-> !(b|c)'::tsquery; - tsquery --------------------------------------- - 'a' & !( 'a' <-> 'b' | 'a' <-> 'c' ) -(1 row) - -SELECT '!(a&b) <-> c'::tsquery; - tsquery --------------------------------------- - !( 'a' <-> 'c' & 'b' <-> 'c' ) & 'c' -(1 row) - -SELECT '!(a|b) <-> c'::tsquery; - tsquery --------------------------------------- - !( 'a' <-> 'c' | 'b' <-> 'c' ) & 'c' -(1 row) - -SELECT '(!a|b) <-> c'::tsquery; - tsquery --------------------------------------- - !( 'a' <-> 'c' ) & 'c' | 'b' <-> 'c' -(1 row) - -SELECT '(!a&b) <-> c'::tsquery; - tsquery --------------------------------------- - !( 'a' <-> 'c' ) & 'c' & 'b' <-> 'c' -(1 row) - -SELECT 'c <-> (!a|b)'::tsquery; - tsquery --------------------------------------- - 'c' & !( 'c' <-> 'a' ) | 'c' <-> 'b' -(1 row) - -SELECT 'c <-> (!a&b)'::tsquery; - tsquery --------------------------------------- - 'c' & !( 'c' <-> 'a' ) & 'c' <-> 'b' -(1 row) - -SELECT '(a|b) <-> !c'::tsquery; - tsquery ------------------------------------------------- - ( 'a' | 'b' ) & !( 'a' <-> 'c' | 'b' <-> 'c' ) -(1 row) - -SELECT '(a&b) <-> !c'::tsquery; - tsquery --------------------------------------------- - 'a' & 'b' & !( 'a' <-> 'c' & 'b' <-> 'c' ) -(1 row) - -SELECT '!c <-> (a|b)'::tsquery; - tsquery -------------------------------------------------- - !( 'c' <-> 'a' ) & 'a' | !( 'c' <-> 'b' ) & 'b' -(1 row) - -SELECT '!c <-> (a&b)'::tsquery; - tsquery -------------------------------------------------- - !( 'c' <-> 'a' ) & 'a' & !( 'c' <-> 'b' ) & 'b' -(1 row) - --comparisons SELECT 'a' < 'b & c'::tsquery as "true"; true @@ -568,33 +441,33 @@ SELECT 'foo & bar'::tsquery && 'asd | fg'; (1 row) SELECT 'a' <-> 'b & d'::tsquery; - ?column? ---------------------------- - 'a' <-> 'b' & 'a' <-> 'd' + ?column? +----------------------- + 'a' <-> ( 'b' & 'd' ) (1 row) SELECT 'a & g' <-> 'b & d'::tsquery; - ?column? -------------------------------------------------------- - 'a' <-> 'b' & 'g' <-> 'b' & 'a' <-> 'd' & 'g' <-> 'd' + ?column? +--------------------------------- + ( 'a' & 'g' ) <-> ( 'b' & 'd' ) (1 row) SELECT 'a & g' <-> 'b | d'::tsquery; - ?column? -------------------------------------------------------- - 'a' <-> 'b' & 'g' <-> 'b' | 'a' <-> 'd' & 'g' <-> 'd' + ?column? +--------------------------------- + ( 'a' & 'g' ) <-> ( 'b' | 'd' ) (1 row) SELECT 'a & g' <-> 'b <-> d'::tsquery; - ?column? ---------------------------------------------------- - 'a' <-> ( 'b' <-> 'd' ) & 'g' <-> ( 'b' <-> 'd' ) + ?column? +----------------------------------- + ( 'a' & 'g' ) <-> ( 'b' <-> 'd' ) (1 row) SELECT tsquery_phrase('a <3> g', 'b & d', 10); - tsquery_phrase ---------------------------------------------- - 'a' <3> 'g' <10> 'b' & 'a' <3> 'g' <10> 'd' + tsquery_phrase +-------------------------------- + 'a' <3> 'g' <10> ( 'b' & 'd' ) (1 row) -- tsvector-tsquery operations @@ -749,25 +622,152 @@ SELECT to_tsvector('simple', '1 2 3 4') @@ '(1 <-> 2) <-> 3' AS "true"; t (1 row) -SELECT to_tsvector('simple', '1 2 3 4') @@ '1 <-> (2 <-> 3)' AS "false"; +SELECT to_tsvector('simple', '1 2 3 4') @@ '1 <-> (2 <-> 3)' AS "true"; + true +------ + t +(1 row) + +SELECT to_tsvector('simple', '1 2 3 4') @@ '1 <2> (2 <-> 3)' AS "false"; false ------- f (1 row) -SELECT to_tsvector('simple', '1 2 3 4') @@ '1 <2> (2 <-> 3)' AS "true"; +SELECT to_tsvector('simple', '1 2 1 2 3 4') @@ '(1 <-> 2) <-> 3' AS "true"; true ------ t (1 row) -SELECT to_tsvector('simple', '1 2 1 2 3 4') @@ '(1 <-> 2) <-> 3' AS "true"; +SELECT to_tsvector('simple', '1 2 1 2 3 4') @@ '1 <-> 2 <-> 3' AS "true"; true ------ t (1 row) -SELECT to_tsvector('simple', '1 2 1 2 3 4') @@ '1 <-> 2 <-> 3' AS "true"; +-- without position data, phrase search does not match +SELECT strip(to_tsvector('simple', '1 2 3 4')) @@ '1 <-> 2 <-> 3' AS "false"; + false +------- + f +(1 row) + +select to_tsvector('simple', 'q x q y') @@ 'q <-> (x & y)' AS "false"; + false +------- + f +(1 row) + +select to_tsvector('simple', 'q x') @@ 'q <-> (x | y <-> z)' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'q y') @@ 'q <-> (x | y <-> z)' AS "false"; + false +------- + f +(1 row) + +select to_tsvector('simple', 'q y z') @@ 'q <-> (x | y <-> z)' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'q y x') @@ 'q <-> (x | y <-> z)' AS "false"; + false +------- + f +(1 row) + +select to_tsvector('simple', 'q x y') @@ 'q <-> (x | y <-> z)' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'q x') @@ '(x | y <-> z) <-> q' AS "false"; + false +------- + f +(1 row) + +select to_tsvector('simple', 'x q') @@ '(x | y <-> z) <-> q' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'x y q') @@ '(x | y <-> z) <-> q' AS "false"; + false +------- + f +(1 row) + +select to_tsvector('simple', 'x y z') @@ '(x | y <-> z) <-> q' AS "false"; + false +------- + f +(1 row) + +select to_tsvector('simple', 'x y z q') @@ '(x | y <-> z) <-> q' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'y z q') @@ '(x | y <-> z) <-> q' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'y y q') @@ '(x | y <-> z) <-> q' AS "false"; + false +------- + f +(1 row) + +select to_tsvector('simple', 'y y q') @@ '(!x | y <-> z) <-> q' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'x y q') @@ '(!x | y <-> z) <-> q' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'y y q') @@ '(x | y <-> !z) <-> q' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'x q') @@ '(x | y <-> !z) <-> q' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'x q') @@ '(!x | y <-> z) <-> q' AS "false"; + false +------- + f +(1 row) + +select to_tsvector('simple', 'z q') @@ '(!x | y <-> z) <-> q' AS "true"; + true +------ + t +(1 row) + +select to_tsvector('simple', 'x y q y') @@ '!x <-> y' AS "true"; true ------ t @@ -1002,6 +1002,12 @@ SELECT 'a:1 b:3'::tsvector @@ 'a <3> b'::tsquery AS "false"; f (1 row) +SELECT 'a:1 b:3'::tsvector @@ 'a <0> a:*'::tsquery AS "true"; + true +------ + t +(1 row) + -- tsvector editing operations SELECT strip('w:12B w:13* w:12,5,6 a:1,3* a:3 w asd:1dc asd'::tsvector); strip diff --git a/src/test/regress/sql/tsearch.sql b/src/test/regress/sql/tsearch.sql index de43860c704..1255f6954db 100644 --- a/src/test/regress/sql/tsearch.sql +++ b/src/test/regress/sql/tsearch.sql @@ -447,6 +447,12 @@ SELECT ts_rewrite( query, 'SELECT keyword, sample FROM test_tsquery' ) FROM to_t SELECT ts_rewrite( query, 'SELECT keyword, sample FROM test_tsquery' ) FROM to_tsquery('english', 'moscow & hotel') AS query; SELECT ts_rewrite( query, 'SELECT keyword, sample FROM test_tsquery' ) FROM to_tsquery('english', 'bar & new & qq & foo & york') AS query; +SELECT ts_rewrite(tsquery_phrase('foo', 'foo'), 'foo', 'bar | baz'); +SELECT to_tsvector('foo bar') @@ + ts_rewrite(tsquery_phrase('foo', 'foo'), 'foo', 'bar | baz'); +SELECT to_tsvector('bar baz') @@ + ts_rewrite(tsquery_phrase('foo', 'foo'), 'foo', 'bar | baz'); + RESET enable_seqscan; --test GUC diff --git a/src/test/regress/sql/tstypes.sql b/src/test/regress/sql/tstypes.sql index 9ea93a29938..77436ce04e0 100644 --- a/src/test/regress/sql/tstypes.sql +++ b/src/test/regress/sql/tstypes.sql @@ -64,34 +64,6 @@ SELECT 'a & !!b'::tsquery; SELECT '!!a & b'::tsquery; SELECT '!!a & !!b'::tsquery; --- phrase transformation -SELECT 'a <-> (b|c)'::tsquery; -SELECT '(a|b) <-> c'::tsquery; -SELECT '(a|b) <-> (d|c)'::tsquery; - -SELECT 'a <-> (b&c)'::tsquery; -SELECT '(a&b) <-> c'::tsquery; -SELECT '(a&b) <-> (d&c)'::tsquery; - -SELECT 'a <-> !b'::tsquery; -SELECT '!a <-> b'::tsquery; -SELECT '!a <-> !b'::tsquery; - -SELECT 'a <-> !(b&c)'::tsquery; -SELECT 'a <-> !(b|c)'::tsquery; -SELECT '!(a&b) <-> c'::tsquery; -SELECT '!(a|b) <-> c'::tsquery; - -SELECT '(!a|b) <-> c'::tsquery; -SELECT '(!a&b) <-> c'::tsquery; -SELECT 'c <-> (!a|b)'::tsquery; -SELECT 'c <-> (!a&b)'::tsquery; - -SELECT '(a|b) <-> !c'::tsquery; -SELECT '(a&b) <-> !c'::tsquery; -SELECT '!c <-> (a|b)'::tsquery; -SELECT '!c <-> (a&b)'::tsquery; - --comparisons SELECT 'a' < 'b & c'::tsquery as "true"; SELECT 'a' > 'b & c'::tsquery as "false"; @@ -146,10 +118,33 @@ SELECT to_tsvector('simple', '1 2 11 3') @@ '1:* <-> 3' AS "true"; SELECT to_tsvector('simple', '1 2 3 4') @@ '1 <-> 2 <-> 3' AS "true"; SELECT to_tsvector('simple', '1 2 3 4') @@ '(1 <-> 2) <-> 3' AS "true"; -SELECT to_tsvector('simple', '1 2 3 4') @@ '1 <-> (2 <-> 3)' AS "false"; -SELECT to_tsvector('simple', '1 2 3 4') @@ '1 <2> (2 <-> 3)' AS "true"; +SELECT to_tsvector('simple', '1 2 3 4') @@ '1 <-> (2 <-> 3)' AS "true"; +SELECT to_tsvector('simple', '1 2 3 4') @@ '1 <2> (2 <-> 3)' AS "false"; SELECT to_tsvector('simple', '1 2 1 2 3 4') @@ '(1 <-> 2) <-> 3' AS "true"; SELECT to_tsvector('simple', '1 2 1 2 3 4') @@ '1 <-> 2 <-> 3' AS "true"; +-- without position data, phrase search does not match +SELECT strip(to_tsvector('simple', '1 2 3 4')) @@ '1 <-> 2 <-> 3' AS "false"; + +select to_tsvector('simple', 'q x q y') @@ 'q <-> (x & y)' AS "false"; +select to_tsvector('simple', 'q x') @@ 'q <-> (x | y <-> z)' AS "true"; +select to_tsvector('simple', 'q y') @@ 'q <-> (x | y <-> z)' AS "false"; +select to_tsvector('simple', 'q y z') @@ 'q <-> (x | y <-> z)' AS "true"; +select to_tsvector('simple', 'q y x') @@ 'q <-> (x | y <-> z)' AS "false"; +select to_tsvector('simple', 'q x y') @@ 'q <-> (x | y <-> z)' AS "true"; +select to_tsvector('simple', 'q x') @@ '(x | y <-> z) <-> q' AS "false"; +select to_tsvector('simple', 'x q') @@ '(x | y <-> z) <-> q' AS "true"; +select to_tsvector('simple', 'x y q') @@ '(x | y <-> z) <-> q' AS "false"; +select to_tsvector('simple', 'x y z') @@ '(x | y <-> z) <-> q' AS "false"; +select to_tsvector('simple', 'x y z q') @@ '(x | y <-> z) <-> q' AS "true"; +select to_tsvector('simple', 'y z q') @@ '(x | y <-> z) <-> q' AS "true"; +select to_tsvector('simple', 'y y q') @@ '(x | y <-> z) <-> q' AS "false"; +select to_tsvector('simple', 'y y q') @@ '(!x | y <-> z) <-> q' AS "true"; +select to_tsvector('simple', 'x y q') @@ '(!x | y <-> z) <-> q' AS "true"; +select to_tsvector('simple', 'y y q') @@ '(x | y <-> !z) <-> q' AS "true"; +select to_tsvector('simple', 'x q') @@ '(x | y <-> !z) <-> q' AS "true"; +select to_tsvector('simple', 'x q') @@ '(!x | y <-> z) <-> q' AS "false"; +select to_tsvector('simple', 'z q') @@ '(!x | y <-> z) <-> q' AS "true"; +select to_tsvector('simple', 'x y q y') @@ '!x <-> y' AS "true"; --ranking SELECT ts_rank(' a:1 s:2C d g'::tsvector, 'a | s'); @@ -193,6 +188,7 @@ SELECT 'a:1 b:3'::tsvector @@ 'a <0> b'::tsquery AS "false"; SELECT 'a:1 b:3'::tsvector @@ 'a <1> b'::tsquery AS "false"; SELECT 'a:1 b:3'::tsvector @@ 'a <2> b'::tsquery AS "true"; SELECT 'a:1 b:3'::tsvector @@ 'a <3> b'::tsquery AS "false"; +SELECT 'a:1 b:3'::tsvector @@ 'a <0> a:*'::tsquery AS "true"; -- tsvector editing operations |
