From 97f3014647a5bd570032abd2b809d3233003f13f Mon Sep 17 00:00:00 2001 From: Teodor Sigaev Date: Mon, 20 Jul 2015 18:18:48 +0300 Subject: This supports the triconsistent function for pg_trgm GIN opclass to make it faster to implement indexed queries where some keys are common and some are rare. Patch by Jeff Janes --- contrib/pg_trgm/Makefile | 2 +- contrib/pg_trgm/pg_trgm--1.1--1.2.sql | 12 +++ contrib/pg_trgm/pg_trgm--1.1.sql | 178 -------------------------------- contrib/pg_trgm/pg_trgm--1.2.sql | 188 ++++++++++++++++++++++++++++++++++ contrib/pg_trgm/pg_trgm.control | 2 +- contrib/pg_trgm/trgm_gin.c | 92 +++++++++++++++++ 6 files changed, 294 insertions(+), 180 deletions(-) create mode 100644 contrib/pg_trgm/pg_trgm--1.1--1.2.sql delete mode 100644 contrib/pg_trgm/pg_trgm--1.1.sql create mode 100644 contrib/pg_trgm/pg_trgm--1.2.sql diff --git a/contrib/pg_trgm/Makefile b/contrib/pg_trgm/Makefile index e081a1e5e9..1e38753622 100644 --- a/contrib/pg_trgm/Makefile +++ b/contrib/pg_trgm/Makefile @@ -4,7 +4,7 @@ MODULE_big = pg_trgm OBJS = trgm_op.o trgm_gist.o trgm_gin.o trgm_regexp.o $(WIN32RES) EXTENSION = pg_trgm -DATA = pg_trgm--1.1.sql pg_trgm--1.0--1.1.sql pg_trgm--unpackaged--1.0.sql +DATA = pg_trgm--1.2.sql pg_trgm--1.0--1.1.sql pg_trgm--1.1--1.2.sql pg_trgm--unpackaged--1.0.sql PGFILEDESC = "pg_trgm - trigram matching" REGRESS = pg_trgm diff --git a/contrib/pg_trgm/pg_trgm--1.1--1.2.sql b/contrib/pg_trgm/pg_trgm--1.1--1.2.sql new file mode 100644 index 0000000000..c101f21061 --- /dev/null +++ b/contrib/pg_trgm/pg_trgm--1.1--1.2.sql @@ -0,0 +1,12 @@ +/* contrib/pg_trgm/pg_trgm--1.1--1.2.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pg_trgm UPDATE TO '1.2'" to load this file. \quit + +CREATE FUNCTION gin_trgm_triconsistent(internal, int2, text, int4, internal, internal, internal) +RETURNS "char" +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +ALTER OPERATOR FAMILY gin_trgm_ops USING gin ADD + FUNCTION 6 (text, text) gin_trgm_triconsistent (internal, int2, text, int4, internal, internal, internal); diff --git a/contrib/pg_trgm/pg_trgm--1.1.sql b/contrib/pg_trgm/pg_trgm--1.1.sql deleted file mode 100644 index 34b37e4787..0000000000 --- a/contrib/pg_trgm/pg_trgm--1.1.sql +++ /dev/null @@ -1,178 +0,0 @@ -/* contrib/pg_trgm/pg_trgm--1.1.sql */ - --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION pg_trgm" to load this file. \quit - -CREATE FUNCTION set_limit(float4) -RETURNS float4 -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT VOLATILE; - -CREATE FUNCTION show_limit() -RETURNS float4 -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT STABLE; - -CREATE FUNCTION show_trgm(text) -RETURNS _text -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT IMMUTABLE; - -CREATE FUNCTION similarity(text,text) -RETURNS float4 -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT IMMUTABLE; - -CREATE FUNCTION similarity_op(text,text) -RETURNS bool -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT STABLE; -- stable because depends on trgm_limit - -CREATE OPERATOR % ( - LEFTARG = text, - RIGHTARG = text, - PROCEDURE = similarity_op, - COMMUTATOR = '%', - RESTRICT = contsel, - JOIN = contjoinsel -); - -CREATE FUNCTION similarity_dist(text,text) -RETURNS float4 -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT IMMUTABLE; - -CREATE OPERATOR <-> ( - LEFTARG = text, - RIGHTARG = text, - PROCEDURE = similarity_dist, - COMMUTATOR = '<->' -); - --- gist key -CREATE FUNCTION gtrgm_in(cstring) -RETURNS gtrgm -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT IMMUTABLE; - -CREATE FUNCTION gtrgm_out(gtrgm) -RETURNS cstring -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT IMMUTABLE; - -CREATE TYPE gtrgm ( - INTERNALLENGTH = -1, - INPUT = gtrgm_in, - OUTPUT = gtrgm_out -); - --- support functions for gist -CREATE FUNCTION gtrgm_consistent(internal,text,int,oid,internal) -RETURNS bool -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - -CREATE FUNCTION gtrgm_distance(internal,text,int,oid) -RETURNS float8 -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - -CREATE FUNCTION gtrgm_compress(internal) -RETURNS internal -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - -CREATE FUNCTION gtrgm_decompress(internal) -RETURNS internal -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - -CREATE FUNCTION gtrgm_penalty(internal,internal,internal) -RETURNS internal -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - -CREATE FUNCTION gtrgm_picksplit(internal, internal) -RETURNS internal -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - -CREATE FUNCTION gtrgm_union(bytea, internal) -RETURNS _int4 -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - -CREATE FUNCTION gtrgm_same(gtrgm, gtrgm, internal) -RETURNS internal -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - --- create the operator class for gist -CREATE OPERATOR CLASS gist_trgm_ops -FOR TYPE text USING gist -AS - OPERATOR 1 % (text, text), - FUNCTION 1 gtrgm_consistent (internal, text, int, oid, internal), - FUNCTION 2 gtrgm_union (bytea, internal), - FUNCTION 3 gtrgm_compress (internal), - FUNCTION 4 gtrgm_decompress (internal), - FUNCTION 5 gtrgm_penalty (internal, internal, internal), - FUNCTION 6 gtrgm_picksplit (internal, internal), - FUNCTION 7 gtrgm_same (gtrgm, gtrgm, internal), - STORAGE gtrgm; - --- Add operators and support functions that are new in 9.1. We do it like --- this, leaving them "loose" in the operator family rather than bound into --- the gist_trgm_ops opclass, because that's the only state that can be --- reproduced during an upgrade from 9.0 (see pg_trgm--unpackaged--1.0.sql). - -ALTER OPERATOR FAMILY gist_trgm_ops USING gist ADD - OPERATOR 2 <-> (text, text) FOR ORDER BY pg_catalog.float_ops, - OPERATOR 3 pg_catalog.~~ (text, text), - OPERATOR 4 pg_catalog.~~* (text, text), - FUNCTION 8 (text, text) gtrgm_distance (internal, text, int, oid); - --- Add operators that are new in 9.3. - -ALTER OPERATOR FAMILY gist_trgm_ops USING gist ADD - OPERATOR 5 pg_catalog.~ (text, text), - OPERATOR 6 pg_catalog.~* (text, text); - --- support functions for gin -CREATE FUNCTION gin_extract_value_trgm(text, internal) -RETURNS internal -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - -CREATE FUNCTION gin_extract_query_trgm(text, internal, int2, internal, internal, internal, internal) -RETURNS internal -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - -CREATE FUNCTION gin_trgm_consistent(internal, int2, text, int4, internal, internal, internal, internal) -RETURNS bool -AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT; - --- create the operator class for gin -CREATE OPERATOR CLASS gin_trgm_ops -FOR TYPE text USING gin -AS - OPERATOR 1 % (text, text), - FUNCTION 1 btint4cmp (int4, int4), - FUNCTION 2 gin_extract_value_trgm (text, internal), - FUNCTION 3 gin_extract_query_trgm (text, internal, int2, internal, internal, internal, internal), - FUNCTION 4 gin_trgm_consistent (internal, int2, text, int4, internal, internal, internal, internal), - STORAGE int4; - --- Add operators that are new in 9.1. - -ALTER OPERATOR FAMILY gin_trgm_ops USING gin ADD - OPERATOR 3 pg_catalog.~~ (text, text), - OPERATOR 4 pg_catalog.~~* (text, text); - --- Add operators that are new in 9.3. - -ALTER OPERATOR FAMILY gin_trgm_ops USING gin ADD - OPERATOR 5 pg_catalog.~ (text, text), - OPERATOR 6 pg_catalog.~* (text, text); diff --git a/contrib/pg_trgm/pg_trgm--1.2.sql b/contrib/pg_trgm/pg_trgm--1.2.sql new file mode 100644 index 0000000000..03d46d07f9 --- /dev/null +++ b/contrib/pg_trgm/pg_trgm--1.2.sql @@ -0,0 +1,188 @@ +/* contrib/pg_trgm/pg_trgm--1.2.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_trgm" to load this file. \quit + +CREATE FUNCTION set_limit(float4) +RETURNS float4 +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT VOLATILE; + +CREATE FUNCTION show_limit() +RETURNS float4 +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT STABLE; + +CREATE FUNCTION show_trgm(text) +RETURNS _text +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT IMMUTABLE; + +CREATE FUNCTION similarity(text,text) +RETURNS float4 +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT IMMUTABLE; + +CREATE FUNCTION similarity_op(text,text) +RETURNS bool +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT STABLE; -- stable because depends on trgm_limit + +CREATE OPERATOR % ( + LEFTARG = text, + RIGHTARG = text, + PROCEDURE = similarity_op, + COMMUTATOR = '%', + RESTRICT = contsel, + JOIN = contjoinsel +); + +CREATE FUNCTION similarity_dist(text,text) +RETURNS float4 +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT IMMUTABLE; + +CREATE OPERATOR <-> ( + LEFTARG = text, + RIGHTARG = text, + PROCEDURE = similarity_dist, + COMMUTATOR = '<->' +); + +-- gist key +CREATE FUNCTION gtrgm_in(cstring) +RETURNS gtrgm +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT IMMUTABLE; + +CREATE FUNCTION gtrgm_out(gtrgm) +RETURNS cstring +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT IMMUTABLE; + +CREATE TYPE gtrgm ( + INTERNALLENGTH = -1, + INPUT = gtrgm_in, + OUTPUT = gtrgm_out +); + +-- support functions for gist +CREATE FUNCTION gtrgm_consistent(internal,text,int,oid,internal) +RETURNS bool +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gtrgm_distance(internal,text,int,oid) +RETURNS float8 +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gtrgm_compress(internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gtrgm_decompress(internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gtrgm_penalty(internal,internal,internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gtrgm_picksplit(internal, internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gtrgm_union(bytea, internal) +RETURNS _int4 +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gtrgm_same(gtrgm, gtrgm, internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +-- create the operator class for gist +CREATE OPERATOR CLASS gist_trgm_ops +FOR TYPE text USING gist +AS + OPERATOR 1 % (text, text), + FUNCTION 1 gtrgm_consistent (internal, text, int, oid, internal), + FUNCTION 2 gtrgm_union (bytea, internal), + FUNCTION 3 gtrgm_compress (internal), + FUNCTION 4 gtrgm_decompress (internal), + FUNCTION 5 gtrgm_penalty (internal, internal, internal), + FUNCTION 6 gtrgm_picksplit (internal, internal), + FUNCTION 7 gtrgm_same (gtrgm, gtrgm, internal), + STORAGE gtrgm; + +-- Add operators and support functions that are new in 9.1. We do it like +-- this, leaving them "loose" in the operator family rather than bound into +-- the gist_trgm_ops opclass, because that's the only state that can be +-- reproduced during an upgrade from 9.0 (see pg_trgm--unpackaged--1.0.sql). + +ALTER OPERATOR FAMILY gist_trgm_ops USING gist ADD + OPERATOR 2 <-> (text, text) FOR ORDER BY pg_catalog.float_ops, + OPERATOR 3 pg_catalog.~~ (text, text), + OPERATOR 4 pg_catalog.~~* (text, text), + FUNCTION 8 (text, text) gtrgm_distance (internal, text, int, oid); + +-- Add operators that are new in 9.3. + +ALTER OPERATOR FAMILY gist_trgm_ops USING gist ADD + OPERATOR 5 pg_catalog.~ (text, text), + OPERATOR 6 pg_catalog.~* (text, text); + +-- support functions for gin +CREATE FUNCTION gin_extract_value_trgm(text, internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gin_extract_query_trgm(text, internal, int2, internal, internal, internal, internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gin_trgm_consistent(internal, int2, text, int4, internal, internal, internal, internal) +RETURNS bool +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +-- create the operator class for gin +CREATE OPERATOR CLASS gin_trgm_ops +FOR TYPE text USING gin +AS + OPERATOR 1 % (text, text), + FUNCTION 1 btint4cmp (int4, int4), + FUNCTION 2 gin_extract_value_trgm (text, internal), + FUNCTION 3 gin_extract_query_trgm (text, internal, int2, internal, internal, internal, internal), + FUNCTION 4 gin_trgm_consistent (internal, int2, text, int4, internal, internal, internal, internal), + STORAGE int4; + +-- Add operators that are new in 9.1. + +ALTER OPERATOR FAMILY gin_trgm_ops USING gin ADD + OPERATOR 3 pg_catalog.~~ (text, text), + OPERATOR 4 pg_catalog.~~* (text, text); + +-- Add operators that are new in 9.3. + +ALTER OPERATOR FAMILY gin_trgm_ops USING gin ADD + OPERATOR 5 pg_catalog.~ (text, text), + OPERATOR 6 pg_catalog.~* (text, text); + +-- Add functions that are new in 9.6 (pg_trgm 1.2). + +CREATE FUNCTION gin_trgm_triconsistent(internal, int2, text, int4, internal, internal, internal) +RETURNS "char" +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +ALTER OPERATOR FAMILY gin_trgm_ops USING gin ADD + FUNCTION 6 (text,text) gin_trgm_triconsistent (internal, int2, text, int4, internal, internal, internal); diff --git a/contrib/pg_trgm/pg_trgm.control b/contrib/pg_trgm/pg_trgm.control index 2ac51e6890..cbf5a186d7 100644 --- a/contrib/pg_trgm/pg_trgm.control +++ b/contrib/pg_trgm/pg_trgm.control @@ -1,5 +1,5 @@ # pg_trgm extension comment = 'text similarity measurement and index searching based on trigrams' -default_version = '1.1' +default_version = '1.2' module_pathname = '$libdir/pg_trgm' relocatable = true diff --git a/contrib/pg_trgm/trgm_gin.c b/contrib/pg_trgm/trgm_gin.c index d524ceaa19..6a0731d44e 100644 --- a/contrib/pg_trgm/trgm_gin.c +++ b/contrib/pg_trgm/trgm_gin.c @@ -14,6 +14,7 @@ PG_FUNCTION_INFO_V1(gin_extract_trgm); PG_FUNCTION_INFO_V1(gin_extract_value_trgm); PG_FUNCTION_INFO_V1(gin_extract_query_trgm); PG_FUNCTION_INFO_V1(gin_trgm_consistent); +PG_FUNCTION_INFO_V1(gin_trgm_triconsistent); /* * This function can only be called if a pre-9.1 version of the GIN operator @@ -235,3 +236,94 @@ gin_trgm_consistent(PG_FUNCTION_ARGS) PG_RETURN_BOOL(res); } + +/* + * In all cases, GIN_TRUE is at least as favorable to inclusion as + * GIN_MAYBE. If no better option is available, simply treat + * GIN_MAYBE as if it were GIN_TRUE and apply the same test as the binary + * consistent function. + */ +Datum +gin_trgm_triconsistent(PG_FUNCTION_ARGS) +{ + GinTernaryValue *check = (GinTernaryValue *) PG_GETARG_POINTER(0); + StrategyNumber strategy = PG_GETARG_UINT16(1); + + /* text *query = PG_GETARG_TEXT_P(2); */ + int32 nkeys = PG_GETARG_INT32(3); + Pointer *extra_data = (Pointer *) PG_GETARG_POINTER(4); + GinTernaryValue res = GIN_MAYBE; + int32 i, + ntrue; + bool *boolcheck; + + switch (strategy) + { + case SimilarityStrategyNumber: + /* Count the matches */ + ntrue = 0; + for (i = 0; i < nkeys; i++) + { + if (check[i] != GIN_FALSE) + ntrue++; + } +#ifdef DIVUNION + res = (nkeys == ntrue) ? GIN_MAYBE : (((((float4) ntrue) / ((float4) (nkeys - ntrue))) >= trgm_limit) ? GIN_MAYBE : GIN_FALSE); +#else + res = (nkeys == 0) ? GIN_FALSE : (((((float4) ntrue) / ((float4) nkeys)) >= trgm_limit) ? GIN_MAYBE : GIN_FALSE); +#endif + break; + case ILikeStrategyNumber: +#ifndef IGNORECASE + elog(ERROR, "cannot handle ~~* with case-sensitive trigrams"); +#endif + /* FALL THRU */ + case LikeStrategyNumber: + /* Check if all extracted trigrams are presented. */ + res = GIN_MAYBE; + for (i = 0; i < nkeys; i++) + { + if (check[i] == GIN_FALSE) + { + res = GIN_FALSE; + break; + } + } + break; + case RegExpICaseStrategyNumber: +#ifndef IGNORECASE + elog(ERROR, "cannot handle ~* with case-sensitive trigrams"); +#endif + /* FALL THRU */ + case RegExpStrategyNumber: + if (nkeys < 1) + { + /* Regex processing gave no result: do full index scan */ + res = GIN_MAYBE; + } + else + { + /* + * As trigramsMatchGraph implements a montonic boolean function, + * promoting all GIN_MAYBE keys to GIN_TRUE will give a + * conservative result. + */ + boolcheck = (bool *) palloc(sizeof(bool) * nkeys); + for (i = 0; i < nkeys; i++) + boolcheck[i] = (check[i] != GIN_FALSE); + if (!trigramsMatchGraph((TrgmPackedGraph *) extra_data[0], + boolcheck)) + res = GIN_FALSE; + pfree(boolcheck); + } + break; + default: + elog(ERROR, "unrecognized strategy number: %d", strategy); + res = GIN_FALSE; /* keep compiler quiet */ + break; + } + + /* All cases served by this function are inexact */ + Assert(res != GIN_TRUE); + PG_RETURN_GIN_TERNARY_VALUE(res); +} -- cgit v1.2.3