* a shared hashtable. (We track only as many distinct queries as will fit
* in the designated amount of shared memory.)
*
- * As of Postgres 9.2, this module normalizes query entries. Normalization
- * is a process whereby similar queries, typically differing only in their
- * constants (though the exact rules are somewhat more subtle than that) are
- * recognized as equivalent, and are tracked as a single entry. This is
- * particularly useful for non-prepared queries.
- *
- * Normalization is implemented by fingerprinting queries, selectively
- * serializing those fields of each query tree's nodes that are judged to be
- * essential to the query. This is referred to as a query jumble. This is
- * distinct from a regular serialization in that various extraneous
- * information is ignored as irrelevant or not essential to the query, such
- * as the collations of Vars and, most notably, the values of constants.
- *
- * This jumble is acquired at the end of parse analysis of each query, and
- * a 64-bit hash of it is stored into the query's Query.queryId field.
- * The server then copies this value around, making it available in plan
- * tree(s) generated from the query. The executor can then use this value
- * to blame query costs on the proper queryId.
+ * Starting in Postgres 9.2, this module normalized query entries. As of
+ * Postgres 14, the normalization is done by the core if compute_query_id is
+ * enabled, or optionally by third-party modules.
*
* To facilitate presenting entries to users, we create "representative" query
* strings in which constants are replaced with parameter symbols ($n), to
#define USAGE_DEALLOC_PERCENT 5 /* free this % of entries at once */
#define IS_STICKY(c) ((c.calls[PGSS_PLAN] + c.calls[PGSS_EXEC]) == 0)
-#define JUMBLE_SIZE 1024 /* query serialization buffer size */
-
/*
* Extension version number, for supporting older extension versions' objects
*/
pgssGlobalStats stats; /* global statistics for pgss */
} pgssSharedState;
-/*
- * Struct for tracking locations/lengths of constants during normalization
- */
-typedef struct pgssLocationLen
-{
- int location; /* start offset in query text */
- int length; /* length in bytes, or -1 to ignore */
-} pgssLocationLen;
-
-/*
- * Working state for computing a query jumble and producing a normalized
- * query string
- */
-typedef struct pgssJumbleState
-{
- /* Jumble of current query tree */
- unsigned char *jumble;
-
- /* Number of bytes used in jumble[] */
- Size jumble_len;
-
- /* Array of locations of constants that should be removed */
- pgssLocationLen *clocations;
-
- /* Allocated length of clocations array */
- int clocations_buf_size;
-
- /* Current number of valid entries in clocations array */
- int clocations_count;
-
- /* highest Param id we've seen, in order to start normalization correctly */
- int highest_extern_param_id;
-} pgssJumbleState;
-
/*---- Local variables ----*/
/* Current nesting depth of ExecutorRun+ProcessUtility calls */
static void pgss_shmem_startup(void);
static void pgss_shmem_shutdown(int code, Datum arg);
-static void pgss_post_parse_analyze(ParseState *pstate, Query *query);
+static void pgss_post_parse_analyze(ParseState *pstate, Query *query,
+ JumbleState *jstate);
static PlannedStmt *pgss_planner(Query *parse,
const char *query_string,
int cursorOptions,
double total_time, uint64 rows,
const BufferUsage *bufusage,
const WalUsage *walusage,
- pgssJumbleState *jstate);
+ JumbleState *jstate);
static void pg_stat_statements_internal(FunctionCallInfo fcinfo,
pgssVersion api_version,
bool showtext);
static bool need_gc_qtexts(void);
static void gc_qtexts(void);
static void entry_reset(Oid userid, Oid dbid, uint64 queryid);
-static void AppendJumble(pgssJumbleState *jstate,
- const unsigned char *item, Size size);
-static void JumbleQuery(pgssJumbleState *jstate, Query *query);
-static void JumbleRangeTable(pgssJumbleState *jstate, List *rtable);
-static void JumbleRowMarks(pgssJumbleState *jstate, List *rowMarks);
-static void JumbleExpr(pgssJumbleState *jstate, Node *node);
-static void RecordConstLocation(pgssJumbleState *jstate, int location);
-static char *generate_normalized_query(pgssJumbleState *jstate, const char *query,
+static char *generate_normalized_query(JumbleState *jstate, const char *query,
int query_loc, int *query_len_p);
-static void fill_in_constant_lengths(pgssJumbleState *jstate, const char *query,
+static void fill_in_constant_lengths(JumbleState *jstate, const char *query,
int query_loc);
static int comp_location(const void *a, const void *b);
* Post-parse-analysis hook: mark query with a queryId
*/
static void
-pgss_post_parse_analyze(ParseState *pstate, Query *query)
+pgss_post_parse_analyze(ParseState *pstate, Query *query, JumbleState *jstate)
{
- pgssJumbleState jstate;
-
if (prev_post_parse_analyze_hook)
- prev_post_parse_analyze_hook(pstate, query);
-
- /* Assert we didn't do this already */
- Assert(query->queryId == UINT64CONST(0));
+ prev_post_parse_analyze_hook(pstate, query, jstate);
/* Safety check... */
if (!pgss || !pgss_hash || !pgss_enabled(exec_nested_level))
return;
}
- /* Set up workspace for query jumbling */
- jstate.jumble = (unsigned char *) palloc(JUMBLE_SIZE);
- jstate.jumble_len = 0;
- jstate.clocations_buf_size = 32;
- jstate.clocations = (pgssLocationLen *)
- palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen));
- jstate.clocations_count = 0;
- jstate.highest_extern_param_id = 0;
-
- /* Compute query ID and mark the Query node with it */
- JumbleQuery(&jstate, query);
- query->queryId =
- DatumGetUInt64(hash_any_extended(jstate.jumble, jstate.jumble_len, 0));
-
/*
- * If we are unlucky enough to get a hash of zero, use 1 instead, to
- * prevent confusion with the utility-statement case.
+ * If query jumbling were able to identify any ignorable constants, we
+ * immediately create a hash table entry for the query, so that we can
+ * record the normalized form of the query string. If there were no such
+ * constants, the normalized string would be the same as the query text
+ * anyway, so there's no need for an early entry.
*/
- if (query->queryId == UINT64CONST(0))
- query->queryId = UINT64CONST(1);
-
- /*
- * If we were able to identify any ignorable constants, we immediately
- * create a hash table entry for the query, so that we can record the
- * normalized form of the query string. If there were no such constants,
- * the normalized string would be the same as the query text anyway, so
- * there's no need for an early entry.
- */
- if (jstate.clocations_count > 0)
+ if (jstate && jstate->clocations_count > 0)
pgss_store(pstate->p_sourcetext,
query->queryId,
query->stmt_location,
0,
NULL,
NULL,
- &jstate);
+ jstate);
}
/*
double total_time, uint64 rows,
const BufferUsage *bufusage,
const WalUsage *walusage,
- pgssJumbleState *jstate)
+ JumbleState *jstate)
{
pgssHashKey key;
pgssEntry *entry;
LWLockRelease(pgss->lock);
}
-/*
- * AppendJumble: Append a value that is substantive in a given query to
- * the current jumble.
- */
-static void
-AppendJumble(pgssJumbleState *jstate, const unsigned char *item, Size size)
-{
- unsigned char *jumble = jstate->jumble;
- Size jumble_len = jstate->jumble_len;
-
- /*
- * Whenever the jumble buffer is full, we hash the current contents and
- * reset the buffer to contain just that hash value, thus relying on the
- * hash to summarize everything so far.
- */
- while (size > 0)
- {
- Size part_size;
-
- if (jumble_len >= JUMBLE_SIZE)
- {
- uint64 start_hash;
-
- start_hash = DatumGetUInt64(hash_any_extended(jumble,
- JUMBLE_SIZE, 0));
- memcpy(jumble, &start_hash, sizeof(start_hash));
- jumble_len = sizeof(start_hash);
- }
- part_size = Min(size, JUMBLE_SIZE - jumble_len);
- memcpy(jumble + jumble_len, item, part_size);
- jumble_len += part_size;
- item += part_size;
- size -= part_size;
- }
- jstate->jumble_len = jumble_len;
-}
-
-/*
- * Wrappers around AppendJumble to encapsulate details of serialization
- * of individual local variable elements.
- */
-#define APP_JUMB(item) \
- AppendJumble(jstate, (const unsigned char *) &(item), sizeof(item))
-#define APP_JUMB_STRING(str) \
- AppendJumble(jstate, (const unsigned char *) (str), strlen(str) + 1)
-
-/*
- * JumbleQuery: Selectively serialize the query tree, appending significant
- * data to the "query jumble" while ignoring nonsignificant data.
- *
- * Rule of thumb for what to include is that we should ignore anything not
- * semantically significant (such as alias names) as well as anything that can
- * be deduced from child nodes (else we'd just be double-hashing that piece
- * of information).
- */
-static void
-JumbleQuery(pgssJumbleState *jstate, Query *query)
-{
- Assert(IsA(query, Query));
- Assert(query->utilityStmt == NULL);
-
- APP_JUMB(query->commandType);
- /* resultRelation is usually predictable from commandType */
- JumbleExpr(jstate, (Node *) query->cteList);
- JumbleRangeTable(jstate, query->rtable);
- JumbleExpr(jstate, (Node *) query->jointree);
- JumbleExpr(jstate, (Node *) query->targetList);
- JumbleExpr(jstate, (Node *) query->onConflict);
- JumbleExpr(jstate, (Node *) query->returningList);
- JumbleExpr(jstate, (Node *) query->groupClause);
- JumbleExpr(jstate, (Node *) query->groupingSets);
- JumbleExpr(jstate, query->havingQual);
- JumbleExpr(jstate, (Node *) query->windowClause);
- JumbleExpr(jstate, (Node *) query->distinctClause);
- JumbleExpr(jstate, (Node *) query->sortClause);
- JumbleExpr(jstate, query->limitOffset);
- JumbleExpr(jstate, query->limitCount);
- JumbleRowMarks(jstate, query->rowMarks);
- JumbleExpr(jstate, query->setOperations);
-}
-
-/*
- * Jumble a range table
- */
-static void
-JumbleRangeTable(pgssJumbleState *jstate, List *rtable)
-{
- ListCell *lc;
-
- foreach(lc, rtable)
- {
- RangeTblEntry *rte = lfirst_node(RangeTblEntry, lc);
-
- APP_JUMB(rte->rtekind);
- switch (rte->rtekind)
- {
- case RTE_RELATION:
- APP_JUMB(rte->relid);
- JumbleExpr(jstate, (Node *) rte->tablesample);
- break;
- case RTE_SUBQUERY:
- JumbleQuery(jstate, rte->subquery);
- break;
- case RTE_JOIN:
- APP_JUMB(rte->jointype);
- break;
- case RTE_FUNCTION:
- JumbleExpr(jstate, (Node *) rte->functions);
- break;
- case RTE_TABLEFUNC:
- JumbleExpr(jstate, (Node *) rte->tablefunc);
- break;
- case RTE_VALUES:
- JumbleExpr(jstate, (Node *) rte->values_lists);
- break;
- case RTE_CTE:
-
- /*
- * Depending on the CTE name here isn't ideal, but it's the
- * only info we have to identify the referenced WITH item.
- */
- APP_JUMB_STRING(rte->ctename);
- APP_JUMB(rte->ctelevelsup);
- break;
- case RTE_NAMEDTUPLESTORE:
- APP_JUMB_STRING(rte->enrname);
- break;
- case RTE_RESULT:
- break;
- default:
- elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind);
- break;
- }
- }
-}
-
-/*
- * Jumble a rowMarks list
- */
-static void
-JumbleRowMarks(pgssJumbleState *jstate, List *rowMarks)
-{
- ListCell *lc;
-
- foreach(lc, rowMarks)
- {
- RowMarkClause *rowmark = lfirst_node(RowMarkClause, lc);
-
- if (!rowmark->pushedDown)
- {
- APP_JUMB(rowmark->rti);
- APP_JUMB(rowmark->strength);
- APP_JUMB(rowmark->waitPolicy);
- }
- }
-}
-
-/*
- * Jumble an expression tree
- *
- * In general this function should handle all the same node types that
- * expression_tree_walker() does, and therefore it's coded to be as parallel
- * to that function as possible. However, since we are only invoked on
- * queries immediately post-parse-analysis, we need not handle node types
- * that only appear in planning.
- *
- * Note: the reason we don't simply use expression_tree_walker() is that the
- * point of that function is to support tree walkers that don't care about
- * most tree node types, but here we care about all types. We should complain
- * about any unrecognized node type.
- */
-static void
-JumbleExpr(pgssJumbleState *jstate, Node *node)
-{
- ListCell *temp;
-
- if (node == NULL)
- return;
-
- /* Guard against stack overflow due to overly complex expressions */
- check_stack_depth();
-
- /*
- * We always emit the node's NodeTag, then any additional fields that are
- * considered significant, and then we recurse to any child nodes.
- */
- APP_JUMB(node->type);
-
- switch (nodeTag(node))
- {
- case T_Var:
- {
- Var *var = (Var *) node;
-
- APP_JUMB(var->varno);
- APP_JUMB(var->varattno);
- APP_JUMB(var->varlevelsup);
- }
- break;
- case T_Const:
- {
- Const *c = (Const *) node;
-
- /* We jumble only the constant's type, not its value */
- APP_JUMB(c->consttype);
- /* Also, record its parse location for query normalization */
- RecordConstLocation(jstate, c->location);
- }
- break;
- case T_Param:
- {
- Param *p = (Param *) node;
-
- APP_JUMB(p->paramkind);
- APP_JUMB(p->paramid);
- APP_JUMB(p->paramtype);
- /* Also, track the highest external Param id */
- if (p->paramkind == PARAM_EXTERN &&
- p->paramid > jstate->highest_extern_param_id)
- jstate->highest_extern_param_id = p->paramid;
- }
- break;
- case T_Aggref:
- {
- Aggref *expr = (Aggref *) node;
-
- APP_JUMB(expr->aggfnoid);
- JumbleExpr(jstate, (Node *) expr->aggdirectargs);
- JumbleExpr(jstate, (Node *) expr->args);
- JumbleExpr(jstate, (Node *) expr->aggorder);
- JumbleExpr(jstate, (Node *) expr->aggdistinct);
- JumbleExpr(jstate, (Node *) expr->aggfilter);
- }
- break;
- case T_GroupingFunc:
- {
- GroupingFunc *grpnode = (GroupingFunc *) node;
-
- JumbleExpr(jstate, (Node *) grpnode->refs);
- }
- break;
- case T_WindowFunc:
- {
- WindowFunc *expr = (WindowFunc *) node;
-
- APP_JUMB(expr->winfnoid);
- APP_JUMB(expr->winref);
- JumbleExpr(jstate, (Node *) expr->args);
- JumbleExpr(jstate, (Node *) expr->aggfilter);
- }
- break;
- case T_SubscriptingRef:
- {
- SubscriptingRef *sbsref = (SubscriptingRef *) node;
-
- JumbleExpr(jstate, (Node *) sbsref->refupperindexpr);
- JumbleExpr(jstate, (Node *) sbsref->reflowerindexpr);
- JumbleExpr(jstate, (Node *) sbsref->refexpr);
- JumbleExpr(jstate, (Node *) sbsref->refassgnexpr);
- }
- break;
- case T_FuncExpr:
- {
- FuncExpr *expr = (FuncExpr *) node;
-
- APP_JUMB(expr->funcid);
- JumbleExpr(jstate, (Node *) expr->args);
- }
- break;
- case T_NamedArgExpr:
- {
- NamedArgExpr *nae = (NamedArgExpr *) node;
-
- APP_JUMB(nae->argnumber);
- JumbleExpr(jstate, (Node *) nae->arg);
- }
- break;
- case T_OpExpr:
- case T_DistinctExpr: /* struct-equivalent to OpExpr */
- case T_NullIfExpr: /* struct-equivalent to OpExpr */
- {
- OpExpr *expr = (OpExpr *) node;
-
- APP_JUMB(expr->opno);
- JumbleExpr(jstate, (Node *) expr->args);
- }
- break;
- case T_ScalarArrayOpExpr:
- {
- ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node;
-
- APP_JUMB(expr->opno);
- APP_JUMB(expr->useOr);
- JumbleExpr(jstate, (Node *) expr->args);
- }
- break;
- case T_BoolExpr:
- {
- BoolExpr *expr = (BoolExpr *) node;
-
- APP_JUMB(expr->boolop);
- JumbleExpr(jstate, (Node *) expr->args);
- }
- break;
- case T_SubLink:
- {
- SubLink *sublink = (SubLink *) node;
-
- APP_JUMB(sublink->subLinkType);
- APP_JUMB(sublink->subLinkId);
- JumbleExpr(jstate, (Node *) sublink->testexpr);
- JumbleQuery(jstate, castNode(Query, sublink->subselect));
- }
- break;
- case T_FieldSelect:
- {
- FieldSelect *fs = (FieldSelect *) node;
-
- APP_JUMB(fs->fieldnum);
- JumbleExpr(jstate, (Node *) fs->arg);
- }
- break;
- case T_FieldStore:
- {
- FieldStore *fstore = (FieldStore *) node;
-
- JumbleExpr(jstate, (Node *) fstore->arg);
- JumbleExpr(jstate, (Node *) fstore->newvals);
- }
- break;
- case T_RelabelType:
- {
- RelabelType *rt = (RelabelType *) node;
-
- APP_JUMB(rt->resulttype);
- JumbleExpr(jstate, (Node *) rt->arg);
- }
- break;
- case T_CoerceViaIO:
- {
- CoerceViaIO *cio = (CoerceViaIO *) node;
-
- APP_JUMB(cio->resulttype);
- JumbleExpr(jstate, (Node *) cio->arg);
- }
- break;
- case T_ArrayCoerceExpr:
- {
- ArrayCoerceExpr *acexpr = (ArrayCoerceExpr *) node;
-
- APP_JUMB(acexpr->resulttype);
- JumbleExpr(jstate, (Node *) acexpr->arg);
- JumbleExpr(jstate, (Node *) acexpr->elemexpr);
- }
- break;
- case T_ConvertRowtypeExpr:
- {
- ConvertRowtypeExpr *crexpr = (ConvertRowtypeExpr *) node;
-
- APP_JUMB(crexpr->resulttype);
- JumbleExpr(jstate, (Node *) crexpr->arg);
- }
- break;
- case T_CollateExpr:
- {
- CollateExpr *ce = (CollateExpr *) node;
-
- APP_JUMB(ce->collOid);
- JumbleExpr(jstate, (Node *) ce->arg);
- }
- break;
- case T_CaseExpr:
- {
- CaseExpr *caseexpr = (CaseExpr *) node;
-
- JumbleExpr(jstate, (Node *) caseexpr->arg);
- foreach(temp, caseexpr->args)
- {
- CaseWhen *when = lfirst_node(CaseWhen, temp);
-
- JumbleExpr(jstate, (Node *) when->expr);
- JumbleExpr(jstate, (Node *) when->result);
- }
- JumbleExpr(jstate, (Node *) caseexpr->defresult);
- }
- break;
- case T_CaseTestExpr:
- {
- CaseTestExpr *ct = (CaseTestExpr *) node;
-
- APP_JUMB(ct->typeId);
- }
- break;
- case T_ArrayExpr:
- JumbleExpr(jstate, (Node *) ((ArrayExpr *) node)->elements);
- break;
- case T_RowExpr:
- JumbleExpr(jstate, (Node *) ((RowExpr *) node)->args);
- break;
- case T_RowCompareExpr:
- {
- RowCompareExpr *rcexpr = (RowCompareExpr *) node;
-
- APP_JUMB(rcexpr->rctype);
- JumbleExpr(jstate, (Node *) rcexpr->largs);
- JumbleExpr(jstate, (Node *) rcexpr->rargs);
- }
- break;
- case T_CoalesceExpr:
- JumbleExpr(jstate, (Node *) ((CoalesceExpr *) node)->args);
- break;
- case T_MinMaxExpr:
- {
- MinMaxExpr *mmexpr = (MinMaxExpr *) node;
-
- APP_JUMB(mmexpr->op);
- JumbleExpr(jstate, (Node *) mmexpr->args);
- }
- break;
- case T_SQLValueFunction:
- {
- SQLValueFunction *svf = (SQLValueFunction *) node;
-
- APP_JUMB(svf->op);
- /* type is fully determined by op */
- APP_JUMB(svf->typmod);
- }
- break;
- case T_XmlExpr:
- {
- XmlExpr *xexpr = (XmlExpr *) node;
-
- APP_JUMB(xexpr->op);
- JumbleExpr(jstate, (Node *) xexpr->named_args);
- JumbleExpr(jstate, (Node *) xexpr->args);
- }
- break;
- case T_NullTest:
- {
- NullTest *nt = (NullTest *) node;
-
- APP_JUMB(nt->nulltesttype);
- JumbleExpr(jstate, (Node *) nt->arg);
- }
- break;
- case T_BooleanTest:
- {
- BooleanTest *bt = (BooleanTest *) node;
-
- APP_JUMB(bt->booltesttype);
- JumbleExpr(jstate, (Node *) bt->arg);
- }
- break;
- case T_CoerceToDomain:
- {
- CoerceToDomain *cd = (CoerceToDomain *) node;
-
- APP_JUMB(cd->resulttype);
- JumbleExpr(jstate, (Node *) cd->arg);
- }
- break;
- case T_CoerceToDomainValue:
- {
- CoerceToDomainValue *cdv = (CoerceToDomainValue *) node;
-
- APP_JUMB(cdv->typeId);
- }
- break;
- case T_SetToDefault:
- {
- SetToDefault *sd = (SetToDefault *) node;
-
- APP_JUMB(sd->typeId);
- }
- break;
- case T_CurrentOfExpr:
- {
- CurrentOfExpr *ce = (CurrentOfExpr *) node;
-
- APP_JUMB(ce->cvarno);
- if (ce->cursor_name)
- APP_JUMB_STRING(ce->cursor_name);
- APP_JUMB(ce->cursor_param);
- }
- break;
- case T_NextValueExpr:
- {
- NextValueExpr *nve = (NextValueExpr *) node;
-
- APP_JUMB(nve->seqid);
- APP_JUMB(nve->typeId);
- }
- break;
- case T_InferenceElem:
- {
- InferenceElem *ie = (InferenceElem *) node;
-
- APP_JUMB(ie->infercollid);
- APP_JUMB(ie->inferopclass);
- JumbleExpr(jstate, ie->expr);
- }
- break;
- case T_TargetEntry:
- {
- TargetEntry *tle = (TargetEntry *) node;
-
- APP_JUMB(tle->resno);
- APP_JUMB(tle->ressortgroupref);
- JumbleExpr(jstate, (Node *) tle->expr);
- }
- break;
- case T_RangeTblRef:
- {
- RangeTblRef *rtr = (RangeTblRef *) node;
-
- APP_JUMB(rtr->rtindex);
- }
- break;
- case T_JoinExpr:
- {
- JoinExpr *join = (JoinExpr *) node;
-
- APP_JUMB(join->jointype);
- APP_JUMB(join->isNatural);
- APP_JUMB(join->rtindex);
- JumbleExpr(jstate, join->larg);
- JumbleExpr(jstate, join->rarg);
- JumbleExpr(jstate, join->quals);
- }
- break;
- case T_FromExpr:
- {
- FromExpr *from = (FromExpr *) node;
-
- JumbleExpr(jstate, (Node *) from->fromlist);
- JumbleExpr(jstate, from->quals);
- }
- break;
- case T_OnConflictExpr:
- {
- OnConflictExpr *conf = (OnConflictExpr *) node;
-
- APP_JUMB(conf->action);
- JumbleExpr(jstate, (Node *) conf->arbiterElems);
- JumbleExpr(jstate, conf->arbiterWhere);
- JumbleExpr(jstate, (Node *) conf->onConflictSet);
- JumbleExpr(jstate, conf->onConflictWhere);
- APP_JUMB(conf->constraint);
- APP_JUMB(conf->exclRelIndex);
- JumbleExpr(jstate, (Node *) conf->exclRelTlist);
- }
- break;
- case T_List:
- foreach(temp, (List *) node)
- {
- JumbleExpr(jstate, (Node *) lfirst(temp));
- }
- break;
- case T_IntList:
- foreach(temp, (List *) node)
- {
- APP_JUMB(lfirst_int(temp));
- }
- break;
- case T_SortGroupClause:
- {
- SortGroupClause *sgc = (SortGroupClause *) node;
-
- APP_JUMB(sgc->tleSortGroupRef);
- APP_JUMB(sgc->eqop);
- APP_JUMB(sgc->sortop);
- APP_JUMB(sgc->nulls_first);
- }
- break;
- case T_GroupingSet:
- {
- GroupingSet *gsnode = (GroupingSet *) node;
-
- JumbleExpr(jstate, (Node *) gsnode->content);
- }
- break;
- case T_WindowClause:
- {
- WindowClause *wc = (WindowClause *) node;
-
- APP_JUMB(wc->winref);
- APP_JUMB(wc->frameOptions);
- JumbleExpr(jstate, (Node *) wc->partitionClause);
- JumbleExpr(jstate, (Node *) wc->orderClause);
- JumbleExpr(jstate, wc->startOffset);
- JumbleExpr(jstate, wc->endOffset);
- }
- break;
- case T_CommonTableExpr:
- {
- CommonTableExpr *cte = (CommonTableExpr *) node;
-
- /* we store the string name because RTE_CTE RTEs need it */
- APP_JUMB_STRING(cte->ctename);
- APP_JUMB(cte->ctematerialized);
- JumbleQuery(jstate, castNode(Query, cte->ctequery));
- }
- break;
- case T_SetOperationStmt:
- {
- SetOperationStmt *setop = (SetOperationStmt *) node;
-
- APP_JUMB(setop->op);
- APP_JUMB(setop->all);
- JumbleExpr(jstate, setop->larg);
- JumbleExpr(jstate, setop->rarg);
- }
- break;
- case T_RangeTblFunction:
- {
- RangeTblFunction *rtfunc = (RangeTblFunction *) node;
-
- JumbleExpr(jstate, rtfunc->funcexpr);
- }
- break;
- case T_TableFunc:
- {
- TableFunc *tablefunc = (TableFunc *) node;
-
- JumbleExpr(jstate, tablefunc->docexpr);
- JumbleExpr(jstate, tablefunc->rowexpr);
- JumbleExpr(jstate, (Node *) tablefunc->colexprs);
- }
- break;
- case T_TableSampleClause:
- {
- TableSampleClause *tsc = (TableSampleClause *) node;
-
- APP_JUMB(tsc->tsmhandler);
- JumbleExpr(jstate, (Node *) tsc->args);
- JumbleExpr(jstate, (Node *) tsc->repeatable);
- }
- break;
- default:
- /* Only a warning, since we can stumble along anyway */
- elog(WARNING, "unrecognized node type: %d",
- (int) nodeTag(node));
- break;
- }
-}
-
-/*
- * Record location of constant within query string of query tree
- * that is currently being walked.
- */
-static void
-RecordConstLocation(pgssJumbleState *jstate, int location)
-{
- /* -1 indicates unknown or undefined location */
- if (location >= 0)
- {
- /* enlarge array if needed */
- if (jstate->clocations_count >= jstate->clocations_buf_size)
- {
- jstate->clocations_buf_size *= 2;
- jstate->clocations = (pgssLocationLen *)
- repalloc(jstate->clocations,
- jstate->clocations_buf_size *
- sizeof(pgssLocationLen));
- }
- jstate->clocations[jstate->clocations_count].location = location;
- /* initialize lengths to -1 to simplify fill_in_constant_lengths */
- jstate->clocations[jstate->clocations_count].length = -1;
- jstate->clocations_count++;
- }
-}
-
/*
* Generate a normalized version of the query string that will be used to
* represent all similar queries.
* Returns a palloc'd string.
*/
static char *
-generate_normalized_query(pgssJumbleState *jstate, const char *query,
+generate_normalized_query(JumbleState *jstate, const char *query,
int query_loc, int *query_len_p)
{
char *norm_query;
* reason for a constant to start with a '-'.
*/
static void
-fill_in_constant_lengths(pgssJumbleState *jstate, const char *query,
+fill_in_constant_lengths(JumbleState *jstate, const char *query,
int query_loc)
{
- pgssLocationLen *locs;
+ LocationLen *locs;
core_yyscan_t yyscanner;
core_yy_extra_type yyextra;
core_YYSTYPE yylval;
*/
if (jstate->clocations_count > 1)
qsort(jstate->clocations, jstate->clocations_count,
- sizeof(pgssLocationLen), comp_location);
+ sizeof(LocationLen), comp_location);
locs = jstate->clocations;
/* initialize the flex scanner --- should match raw_parser() */
}
/*
- * comp_location: comparator for qsorting pgssLocationLen structs by location
+ * comp_location: comparator for qsorting LocationLen structs by location
*/
static int
comp_location(const void *a, const void *b)
{
- int l = ((const pgssLocationLen *) a)->location;
- int r = ((const pgssLocationLen *) b)->location;
+ int l = ((const LocationLen *) a)->location;
+ int r = ((const LocationLen *) b)->location;
if (l < r)
return -1;
shared_preload_libraries = 'pg_stat_statements'
+compute_query_id = on
<title>Statistics Monitoring</title>
<variablelist>
+ <varlistentry id="guc-compute-query-id" xreflabel="compute_query_id">
+ <term><varname>compute_query_id</varname> (<type>boolean</type>)
+ <indexterm>
+ <primary><varname>compute_query_id</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Enables in-core computation of a query identifier. The <xref
+ linkend="pgstatstatements"/> extension requires a query identifier
+ to be computed. Note that an external module can alternatively
+ be used if the in-core query identifier computation method
+ isn't acceptable. In this case, in-core computation should
+ remain disabled. The default is <literal>off</literal>.
+ </para>
+ <note>
+ <para>
+ To ensure that a only one query identifier is calculated and
+ displayed, extensions that calculate query identifiers should
+ throw an error if a query identifier has already been computed.
+ </para>
+ </note>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><varname>log_statement_stats</varname> (<type>boolean</type>)
<indexterm>
This means that a server restart is needed to add or remove the module.
</para>
+ <para>
+ The module will not track statistics unless query
+ identifiers are calculated. This can be done by enabling <xref
+ linkend="guc-compute-query-id"/> or using a third-party module that
+ computes its own query identifiers. Note that all statistics tracked
+ by this module must be reset if the query identifier method is changed.
+ </para>
+
<para>
When <filename>pg_stat_statements</filename> is loaded, it tracks
statistics across all databases of the server. To access and manipulate
<structfield>queryid</structfield> <type>bigint</type>
</para>
<para>
- Internal hash code, computed from the statement's parse tree
+ Hash code to identify identical normalized queries.
</para></entry>
</row>
are compared strictly on the basis of their textual query strings, however.
</para>
+ <note>
+ <para>
+ The following details about constant replacement and
+ <structfield>queryid</structfield> only applies when <xref
+ linkend="guc-compute-query-id"/> is enabled. If you use an external
+ module instead to compute <structfield>queryid</structfield>, you
+ should refer to its documentation for details.
+ </para>
+ </note>
+
<para>
When a constant's value has been ignored for purposes of matching the query
to other queries, the constant is replaced by a parameter symbol, such
#include "parser/parsetree.h"
#include "rewrite/rewriteManip.h"
#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/queryjumble.h"
#include "utils/rel.h"
{
ParseState *pstate = make_parsestate(NULL);
Query *query;
+ JumbleState *jstate = NULL;
Assert(sourceText != NULL); /* required as of 8.4 */
query = transformTopLevelStmt(pstate, parseTree);
+ if (compute_query_id)
+ jstate = JumbleQuery(query, sourceText);
+
if (post_parse_analyze_hook)
- (*post_parse_analyze_hook) (pstate, query);
+ (*post_parse_analyze_hook) (pstate, query, jstate);
free_parsestate(pstate);
{
ParseState *pstate = make_parsestate(NULL);
Query *query;
+ JumbleState *jstate = NULL;
Assert(sourceText != NULL); /* required as of 8.4 */
/* make sure all is well with parameter types */
check_variable_parameters(pstate, query);
+ if (compute_query_id)
+ jstate = JumbleQuery(query, sourceText);
+
if (post_parse_analyze_hook)
- (*post_parse_analyze_hook) (pstate, query);
+ (*post_parse_analyze_hook) (pstate, query, jstate);
free_parsestate(pstate);
ParseState *pstate;
Query *query;
List *querytree_list;
+ JumbleState *jstate = NULL;
Assert(query_string != NULL); /* required as of 8.4 */
query = transformTopLevelStmt(pstate, parsetree);
+ if (compute_query_id)
+ jstate = JumbleQuery(query, query_string);
+
if (post_parse_analyze_hook)
- (*post_parse_analyze_hook) (pstate, query);
+ (*post_parse_analyze_hook) (pstate, query, jstate);
free_parsestate(pstate);
pg_rusage.o \
ps_status.o \
queryenvironment.o \
+ queryjumble.o \
rls.o \
sampling.o \
superuser.o \
/*
* GUC option variables that are exported from this module
*/
+bool compute_query_id = false;
bool log_duration = false;
bool Debug_print_plan = false;
bool Debug_print_parse = false;
true,
NULL, NULL, NULL
},
+ {
+ {"compute_query_id", PGC_SUSET, STATS_MONITORING,
+ gettext_noop("Compute query identifiers."),
+ NULL
+ },
+ &compute_query_id,
+ false,
+ NULL, NULL, NULL
+ },
{
{"log_parser_stats", PGC_SUSET, STATS_MONITORING,
gettext_noop("Writes parser performance statistics to the server log."),
# - Monitoring -
+#compute_query_id = off
#log_parser_stats = off
#log_planner_stats = off
#log_executor_stats = off
--- /dev/null
+/*-------------------------------------------------------------------------
+ *
+ * queryjumble.c
+ * Query normalization and fingerprinting.
+ *
+ * Normalization is a process whereby similar queries, typically differing only
+ * in their constants (though the exact rules are somewhat more subtle than
+ * that) are recognized as equivalent, and are tracked as a single entry. This
+ * is particularly useful for non-prepared queries.
+ *
+ * Normalization is implemented by fingerprinting queries, selectively
+ * serializing those fields of each query tree's nodes that are judged to be
+ * essential to the query. This is referred to as a query jumble. This is
+ * distinct from a regular serialization in that various extraneous
+ * information is ignored as irrelevant or not essential to the query, such
+ * as the collations of Vars and, most notably, the values of constants.
+ *
+ * This jumble is acquired at the end of parse analysis of each query, and
+ * a 64-bit hash of it is stored into the query's Query.queryId field.
+ * The server then copies this value around, making it available in plan
+ * tree(s) generated from the query. The executor can then use this value
+ * to blame query costs on the proper queryId.
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/utils/misc/queryjumble.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "common/hashfn.h"
+#include "miscadmin.h"
+#include "parser/scansup.h"
+#include "utils/queryjumble.h"
+
+#define JUMBLE_SIZE 1024 /* query serialization buffer size */
+
+static uint64 compute_utility_queryid(const char *str, int query_len);
+static void AppendJumble(JumbleState *jstate,
+ const unsigned char *item, Size size);
+static void JumbleQueryInternal(JumbleState *jstate, Query *query);
+static void JumbleRangeTable(JumbleState *jstate, List *rtable);
+static void JumbleRowMarks(JumbleState *jstate, List *rowMarks);
+static void JumbleExpr(JumbleState *jstate, Node *node);
+static void RecordConstLocation(JumbleState *jstate, int location);
+
+/*
+ * Given a possibly multi-statement source string, confine our attention to the
+ * relevant part of the string.
+ */
+const char *
+CleanQuerytext(const char *query, int *location, int *len)
+{
+ int query_location = *location;
+ int query_len = *len;
+
+ /* First apply starting offset, unless it's -1 (unknown). */
+ if (query_location >= 0)
+ {
+ Assert(query_location <= strlen(query));
+ query += query_location;
+ /* Length of 0 (or -1) means "rest of string" */
+ if (query_len <= 0)
+ query_len = strlen(query);
+ else
+ Assert(query_len <= strlen(query));
+ }
+ else
+ {
+ /* If query location is unknown, distrust query_len as well */
+ query_location = 0;
+ query_len = strlen(query);
+ }
+
+ /*
+ * Discard leading and trailing whitespace, too. Use scanner_isspace()
+ * not libc's isspace(), because we want to match the lexer's behavior.
+ */
+ while (query_len > 0 && scanner_isspace(query[0]))
+ query++, query_location++, query_len--;
+ while (query_len > 0 && scanner_isspace(query[query_len - 1]))
+ query_len--;
+
+ *location = query_location;
+ *len = query_len;
+
+ return query;
+}
+
+JumbleState *
+JumbleQuery(Query *query, const char *querytext)
+{
+ JumbleState *jstate = NULL;
+ if (query->utilityStmt)
+ {
+ const char *sql;
+ int query_location = query->stmt_location;
+ int query_len = query->stmt_len;
+
+ /*
+ * Confine our attention to the relevant part of the string, if the
+ * query is a portion of a multi-statement source string.
+ */
+ sql = CleanQuerytext(querytext, &query_location, &query_len);
+
+ query->queryId = compute_utility_queryid(sql, query_len);
+ }
+ else
+ {
+ jstate = (JumbleState *) palloc(sizeof(JumbleState));
+
+ /* Set up workspace for query jumbling */
+ jstate->jumble = (unsigned char *) palloc(JUMBLE_SIZE);
+ jstate->jumble_len = 0;
+ jstate->clocations_buf_size = 32;
+ jstate->clocations = (LocationLen *)
+ palloc(jstate->clocations_buf_size * sizeof(LocationLen));
+ jstate->clocations_count = 0;
+ jstate->highest_extern_param_id = 0;
+
+ /* Compute query ID and mark the Query node with it */
+ JumbleQueryInternal(jstate, query);
+ query->queryId = DatumGetUInt64(hash_any_extended(jstate->jumble,
+ jstate->jumble_len,
+ 0));
+
+ /*
+ * If we are unlucky enough to get a hash of zero, use 1 instead, to
+ * prevent confusion with the utility-statement case.
+ */
+ if (query->queryId == UINT64CONST(0))
+ query->queryId = UINT64CONST(1);
+ }
+
+ return jstate;
+}
+
+/*
+ * Compute a query identifier for the given utility query string.
+ */
+static uint64
+compute_utility_queryid(const char *str, int query_len)
+{
+ uint64 queryId;
+
+ queryId = DatumGetUInt64(hash_any_extended((const unsigned char *) str,
+ query_len, 0));
+
+ /*
+ * If we are unlucky enough to get a hash of zero(invalid), use
+ * queryID as 2 instead, queryID 1 is already in use for normal
+ * statements.
+ */
+ if (queryId == UINT64CONST(0))
+ queryId = UINT64CONST(2);
+
+ return queryId;
+}
+
+/*
+ * AppendJumble: Append a value that is substantive in a given query to
+ * the current jumble.
+ */
+static void
+AppendJumble(JumbleState *jstate, const unsigned char *item, Size size)
+{
+ unsigned char *jumble = jstate->jumble;
+ Size jumble_len = jstate->jumble_len;
+
+ /*
+ * Whenever the jumble buffer is full, we hash the current contents and
+ * reset the buffer to contain just that hash value, thus relying on the
+ * hash to summarize everything so far.
+ */
+ while (size > 0)
+ {
+ Size part_size;
+
+ if (jumble_len >= JUMBLE_SIZE)
+ {
+ uint64 start_hash;
+
+ start_hash = DatumGetUInt64(hash_any_extended(jumble,
+ JUMBLE_SIZE, 0));
+ memcpy(jumble, &start_hash, sizeof(start_hash));
+ jumble_len = sizeof(start_hash);
+ }
+ part_size = Min(size, JUMBLE_SIZE - jumble_len);
+ memcpy(jumble + jumble_len, item, part_size);
+ jumble_len += part_size;
+ item += part_size;
+ size -= part_size;
+ }
+ jstate->jumble_len = jumble_len;
+}
+
+/*
+ * Wrappers around AppendJumble to encapsulate details of serialization
+ * of individual local variable elements.
+ */
+#define APP_JUMB(item) \
+ AppendJumble(jstate, (const unsigned char *) &(item), sizeof(item))
+#define APP_JUMB_STRING(str) \
+ AppendJumble(jstate, (const unsigned char *) (str), strlen(str) + 1)
+
+/*
+ * JumbleQueryInternal: Selectively serialize the query tree, appending
+ * significant data to the "query jumble" while ignoring nonsignificant data.
+ *
+ * Rule of thumb for what to include is that we should ignore anything not
+ * semantically significant (such as alias names) as well as anything that can
+ * be deduced from child nodes (else we'd just be double-hashing that piece
+ * of information).
+ */
+static void
+JumbleQueryInternal(JumbleState *jstate, Query *query)
+{
+ Assert(IsA(query, Query));
+ Assert(query->utilityStmt == NULL);
+
+ APP_JUMB(query->commandType);
+ /* resultRelation is usually predictable from commandType */
+ JumbleExpr(jstate, (Node *) query->cteList);
+ JumbleRangeTable(jstate, query->rtable);
+ JumbleExpr(jstate, (Node *) query->jointree);
+ JumbleExpr(jstate, (Node *) query->targetList);
+ JumbleExpr(jstate, (Node *) query->onConflict);
+ JumbleExpr(jstate, (Node *) query->returningList);
+ JumbleExpr(jstate, (Node *) query->groupClause);
+ JumbleExpr(jstate, (Node *) query->groupingSets);
+ JumbleExpr(jstate, query->havingQual);
+ JumbleExpr(jstate, (Node *) query->windowClause);
+ JumbleExpr(jstate, (Node *) query->distinctClause);
+ JumbleExpr(jstate, (Node *) query->sortClause);
+ JumbleExpr(jstate, query->limitOffset);
+ JumbleExpr(jstate, query->limitCount);
+ JumbleRowMarks(jstate, query->rowMarks);
+ JumbleExpr(jstate, query->setOperations);
+}
+
+/*
+ * Jumble a range table
+ */
+static void
+JumbleRangeTable(JumbleState *jstate, List *rtable)
+{
+ ListCell *lc;
+
+ foreach(lc, rtable)
+ {
+ RangeTblEntry *rte = lfirst_node(RangeTblEntry, lc);
+
+ APP_JUMB(rte->rtekind);
+ switch (rte->rtekind)
+ {
+ case RTE_RELATION:
+ APP_JUMB(rte->relid);
+ JumbleExpr(jstate, (Node *) rte->tablesample);
+ break;
+ case RTE_SUBQUERY:
+ JumbleQueryInternal(jstate, rte->subquery);
+ break;
+ case RTE_JOIN:
+ APP_JUMB(rte->jointype);
+ break;
+ case RTE_FUNCTION:
+ JumbleExpr(jstate, (Node *) rte->functions);
+ break;
+ case RTE_TABLEFUNC:
+ JumbleExpr(jstate, (Node *) rte->tablefunc);
+ break;
+ case RTE_VALUES:
+ JumbleExpr(jstate, (Node *) rte->values_lists);
+ break;
+ case RTE_CTE:
+
+ /*
+ * Depending on the CTE name here isn't ideal, but it's the
+ * only info we have to identify the referenced WITH item.
+ */
+ APP_JUMB_STRING(rte->ctename);
+ APP_JUMB(rte->ctelevelsup);
+ break;
+ case RTE_NAMEDTUPLESTORE:
+ APP_JUMB_STRING(rte->enrname);
+ break;
+ case RTE_RESULT:
+ break;
+ default:
+ elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind);
+ break;
+ }
+ }
+}
+
+/*
+ * Jumble a rowMarks list
+ */
+static void
+JumbleRowMarks(JumbleState *jstate, List *rowMarks)
+{
+ ListCell *lc;
+
+ foreach(lc, rowMarks)
+ {
+ RowMarkClause *rowmark = lfirst_node(RowMarkClause, lc);
+
+ if (!rowmark->pushedDown)
+ {
+ APP_JUMB(rowmark->rti);
+ APP_JUMB(rowmark->strength);
+ APP_JUMB(rowmark->waitPolicy);
+ }
+ }
+}
+
+/*
+ * Jumble an expression tree
+ *
+ * In general this function should handle all the same node types that
+ * expression_tree_walker() does, and therefore it's coded to be as parallel
+ * to that function as possible. However, since we are only invoked on
+ * queries immediately post-parse-analysis, we need not handle node types
+ * that only appear in planning.
+ *
+ * Note: the reason we don't simply use expression_tree_walker() is that the
+ * point of that function is to support tree walkers that don't care about
+ * most tree node types, but here we care about all types. We should complain
+ * about any unrecognized node type.
+ */
+static void
+JumbleExpr(JumbleState *jstate, Node *node)
+{
+ ListCell *temp;
+
+ if (node == NULL)
+ return;
+
+ /* Guard against stack overflow due to overly complex expressions */
+ check_stack_depth();
+
+ /*
+ * We always emit the node's NodeTag, then any additional fields that are
+ * considered significant, and then we recurse to any child nodes.
+ */
+ APP_JUMB(node->type);
+
+ switch (nodeTag(node))
+ {
+ case T_Var:
+ {
+ Var *var = (Var *) node;
+
+ APP_JUMB(var->varno);
+ APP_JUMB(var->varattno);
+ APP_JUMB(var->varlevelsup);
+ }
+ break;
+ case T_Const:
+ {
+ Const *c = (Const *) node;
+
+ /* We jumble only the constant's type, not its value */
+ APP_JUMB(c->consttype);
+ /* Also, record its parse location for query normalization */
+ RecordConstLocation(jstate, c->location);
+ }
+ break;
+ case T_Param:
+ {
+ Param *p = (Param *) node;
+
+ APP_JUMB(p->paramkind);
+ APP_JUMB(p->paramid);
+ APP_JUMB(p->paramtype);
+ /* Also, track the highest external Param id */
+ if (p->paramkind == PARAM_EXTERN &&
+ p->paramid > jstate->highest_extern_param_id)
+ jstate->highest_extern_param_id = p->paramid;
+ }
+ break;
+ case T_Aggref:
+ {
+ Aggref *expr = (Aggref *) node;
+
+ APP_JUMB(expr->aggfnoid);
+ JumbleExpr(jstate, (Node *) expr->aggdirectargs);
+ JumbleExpr(jstate, (Node *) expr->args);
+ JumbleExpr(jstate, (Node *) expr->aggorder);
+ JumbleExpr(jstate, (Node *) expr->aggdistinct);
+ JumbleExpr(jstate, (Node *) expr->aggfilter);
+ }
+ break;
+ case T_GroupingFunc:
+ {
+ GroupingFunc *grpnode = (GroupingFunc *) node;
+
+ JumbleExpr(jstate, (Node *) grpnode->refs);
+ }
+ break;
+ case T_WindowFunc:
+ {
+ WindowFunc *expr = (WindowFunc *) node;
+
+ APP_JUMB(expr->winfnoid);
+ APP_JUMB(expr->winref);
+ JumbleExpr(jstate, (Node *) expr->args);
+ JumbleExpr(jstate, (Node *) expr->aggfilter);
+ }
+ break;
+ case T_SubscriptingRef:
+ {
+ SubscriptingRef *sbsref = (SubscriptingRef *) node;
+
+ JumbleExpr(jstate, (Node *) sbsref->refupperindexpr);
+ JumbleExpr(jstate, (Node *) sbsref->reflowerindexpr);
+ JumbleExpr(jstate, (Node *) sbsref->refexpr);
+ JumbleExpr(jstate, (Node *) sbsref->refassgnexpr);
+ }
+ break;
+ case T_FuncExpr:
+ {
+ FuncExpr *expr = (FuncExpr *) node;
+
+ APP_JUMB(expr->funcid);
+ JumbleExpr(jstate, (Node *) expr->args);
+ }
+ break;
+ case T_NamedArgExpr:
+ {
+ NamedArgExpr *nae = (NamedArgExpr *) node;
+
+ APP_JUMB(nae->argnumber);
+ JumbleExpr(jstate, (Node *) nae->arg);
+ }
+ break;
+ case T_OpExpr:
+ case T_DistinctExpr: /* struct-equivalent to OpExpr */
+ case T_NullIfExpr: /* struct-equivalent to OpExpr */
+ {
+ OpExpr *expr = (OpExpr *) node;
+
+ APP_JUMB(expr->opno);
+ JumbleExpr(jstate, (Node *) expr->args);
+ }
+ break;
+ case T_ScalarArrayOpExpr:
+ {
+ ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node;
+
+ APP_JUMB(expr->opno);
+ APP_JUMB(expr->useOr);
+ JumbleExpr(jstate, (Node *) expr->args);
+ }
+ break;
+ case T_BoolExpr:
+ {
+ BoolExpr *expr = (BoolExpr *) node;
+
+ APP_JUMB(expr->boolop);
+ JumbleExpr(jstate, (Node *) expr->args);
+ }
+ break;
+ case T_SubLink:
+ {
+ SubLink *sublink = (SubLink *) node;
+
+ APP_JUMB(sublink->subLinkType);
+ APP_JUMB(sublink->subLinkId);
+ JumbleExpr(jstate, (Node *) sublink->testexpr);
+ JumbleQueryInternal(jstate, castNode(Query, sublink->subselect));
+ }
+ break;
+ case T_FieldSelect:
+ {
+ FieldSelect *fs = (FieldSelect *) node;
+
+ APP_JUMB(fs->fieldnum);
+ JumbleExpr(jstate, (Node *) fs->arg);
+ }
+ break;
+ case T_FieldStore:
+ {
+ FieldStore *fstore = (FieldStore *) node;
+
+ JumbleExpr(jstate, (Node *) fstore->arg);
+ JumbleExpr(jstate, (Node *) fstore->newvals);
+ }
+ break;
+ case T_RelabelType:
+ {
+ RelabelType *rt = (RelabelType *) node;
+
+ APP_JUMB(rt->resulttype);
+ JumbleExpr(jstate, (Node *) rt->arg);
+ }
+ break;
+ case T_CoerceViaIO:
+ {
+ CoerceViaIO *cio = (CoerceViaIO *) node;
+
+ APP_JUMB(cio->resulttype);
+ JumbleExpr(jstate, (Node *) cio->arg);
+ }
+ break;
+ case T_ArrayCoerceExpr:
+ {
+ ArrayCoerceExpr *acexpr = (ArrayCoerceExpr *) node;
+
+ APP_JUMB(acexpr->resulttype);
+ JumbleExpr(jstate, (Node *) acexpr->arg);
+ JumbleExpr(jstate, (Node *) acexpr->elemexpr);
+ }
+ break;
+ case T_ConvertRowtypeExpr:
+ {
+ ConvertRowtypeExpr *crexpr = (ConvertRowtypeExpr *) node;
+
+ APP_JUMB(crexpr->resulttype);
+ JumbleExpr(jstate, (Node *) crexpr->arg);
+ }
+ break;
+ case T_CollateExpr:
+ {
+ CollateExpr *ce = (CollateExpr *) node;
+
+ APP_JUMB(ce->collOid);
+ JumbleExpr(jstate, (Node *) ce->arg);
+ }
+ break;
+ case T_CaseExpr:
+ {
+ CaseExpr *caseexpr = (CaseExpr *) node;
+
+ JumbleExpr(jstate, (Node *) caseexpr->arg);
+ foreach(temp, caseexpr->args)
+ {
+ CaseWhen *when = lfirst_node(CaseWhen, temp);
+
+ JumbleExpr(jstate, (Node *) when->expr);
+ JumbleExpr(jstate, (Node *) when->result);
+ }
+ JumbleExpr(jstate, (Node *) caseexpr->defresult);
+ }
+ break;
+ case T_CaseTestExpr:
+ {
+ CaseTestExpr *ct = (CaseTestExpr *) node;
+
+ APP_JUMB(ct->typeId);
+ }
+ break;
+ case T_ArrayExpr:
+ JumbleExpr(jstate, (Node *) ((ArrayExpr *) node)->elements);
+ break;
+ case T_RowExpr:
+ JumbleExpr(jstate, (Node *) ((RowExpr *) node)->args);
+ break;
+ case T_RowCompareExpr:
+ {
+ RowCompareExpr *rcexpr = (RowCompareExpr *) node;
+
+ APP_JUMB(rcexpr->rctype);
+ JumbleExpr(jstate, (Node *) rcexpr->largs);
+ JumbleExpr(jstate, (Node *) rcexpr->rargs);
+ }
+ break;
+ case T_CoalesceExpr:
+ JumbleExpr(jstate, (Node *) ((CoalesceExpr *) node)->args);
+ break;
+ case T_MinMaxExpr:
+ {
+ MinMaxExpr *mmexpr = (MinMaxExpr *) node;
+
+ APP_JUMB(mmexpr->op);
+ JumbleExpr(jstate, (Node *) mmexpr->args);
+ }
+ break;
+ case T_SQLValueFunction:
+ {
+ SQLValueFunction *svf = (SQLValueFunction *) node;
+
+ APP_JUMB(svf->op);
+ /* type is fully determined by op */
+ APP_JUMB(svf->typmod);
+ }
+ break;
+ case T_XmlExpr:
+ {
+ XmlExpr *xexpr = (XmlExpr *) node;
+
+ APP_JUMB(xexpr->op);
+ JumbleExpr(jstate, (Node *) xexpr->named_args);
+ JumbleExpr(jstate, (Node *) xexpr->args);
+ }
+ break;
+ case T_NullTest:
+ {
+ NullTest *nt = (NullTest *) node;
+
+ APP_JUMB(nt->nulltesttype);
+ JumbleExpr(jstate, (Node *) nt->arg);
+ }
+ break;
+ case T_BooleanTest:
+ {
+ BooleanTest *bt = (BooleanTest *) node;
+
+ APP_JUMB(bt->booltesttype);
+ JumbleExpr(jstate, (Node *) bt->arg);
+ }
+ break;
+ case T_CoerceToDomain:
+ {
+ CoerceToDomain *cd = (CoerceToDomain *) node;
+
+ APP_JUMB(cd->resulttype);
+ JumbleExpr(jstate, (Node *) cd->arg);
+ }
+ break;
+ case T_CoerceToDomainValue:
+ {
+ CoerceToDomainValue *cdv = (CoerceToDomainValue *) node;
+
+ APP_JUMB(cdv->typeId);
+ }
+ break;
+ case T_SetToDefault:
+ {
+ SetToDefault *sd = (SetToDefault *) node;
+
+ APP_JUMB(sd->typeId);
+ }
+ break;
+ case T_CurrentOfExpr:
+ {
+ CurrentOfExpr *ce = (CurrentOfExpr *) node;
+
+ APP_JUMB(ce->cvarno);
+ if (ce->cursor_name)
+ APP_JUMB_STRING(ce->cursor_name);
+ APP_JUMB(ce->cursor_param);
+ }
+ break;
+ case T_NextValueExpr:
+ {
+ NextValueExpr *nve = (NextValueExpr *) node;
+
+ APP_JUMB(nve->seqid);
+ APP_JUMB(nve->typeId);
+ }
+ break;
+ case T_InferenceElem:
+ {
+ InferenceElem *ie = (InferenceElem *) node;
+
+ APP_JUMB(ie->infercollid);
+ APP_JUMB(ie->inferopclass);
+ JumbleExpr(jstate, ie->expr);
+ }
+ break;
+ case T_TargetEntry:
+ {
+ TargetEntry *tle = (TargetEntry *) node;
+
+ APP_JUMB(tle->resno);
+ APP_JUMB(tle->ressortgroupref);
+ JumbleExpr(jstate, (Node *) tle->expr);
+ }
+ break;
+ case T_RangeTblRef:
+ {
+ RangeTblRef *rtr = (RangeTblRef *) node;
+
+ APP_JUMB(rtr->rtindex);
+ }
+ break;
+ case T_JoinExpr:
+ {
+ JoinExpr *join = (JoinExpr *) node;
+
+ APP_JUMB(join->jointype);
+ APP_JUMB(join->isNatural);
+ APP_JUMB(join->rtindex);
+ JumbleExpr(jstate, join->larg);
+ JumbleExpr(jstate, join->rarg);
+ JumbleExpr(jstate, join->quals);
+ }
+ break;
+ case T_FromExpr:
+ {
+ FromExpr *from = (FromExpr *) node;
+
+ JumbleExpr(jstate, (Node *) from->fromlist);
+ JumbleExpr(jstate, from->quals);
+ }
+ break;
+ case T_OnConflictExpr:
+ {
+ OnConflictExpr *conf = (OnConflictExpr *) node;
+
+ APP_JUMB(conf->action);
+ JumbleExpr(jstate, (Node *) conf->arbiterElems);
+ JumbleExpr(jstate, conf->arbiterWhere);
+ JumbleExpr(jstate, (Node *) conf->onConflictSet);
+ JumbleExpr(jstate, conf->onConflictWhere);
+ APP_JUMB(conf->constraint);
+ APP_JUMB(conf->exclRelIndex);
+ JumbleExpr(jstate, (Node *) conf->exclRelTlist);
+ }
+ break;
+ case T_List:
+ foreach(temp, (List *) node)
+ {
+ JumbleExpr(jstate, (Node *) lfirst(temp));
+ }
+ break;
+ case T_IntList:
+ foreach(temp, (List *) node)
+ {
+ APP_JUMB(lfirst_int(temp));
+ }
+ break;
+ case T_SortGroupClause:
+ {
+ SortGroupClause *sgc = (SortGroupClause *) node;
+
+ APP_JUMB(sgc->tleSortGroupRef);
+ APP_JUMB(sgc->eqop);
+ APP_JUMB(sgc->sortop);
+ APP_JUMB(sgc->nulls_first);
+ }
+ break;
+ case T_GroupingSet:
+ {
+ GroupingSet *gsnode = (GroupingSet *) node;
+
+ JumbleExpr(jstate, (Node *) gsnode->content);
+ }
+ break;
+ case T_WindowClause:
+ {
+ WindowClause *wc = (WindowClause *) node;
+
+ APP_JUMB(wc->winref);
+ APP_JUMB(wc->frameOptions);
+ JumbleExpr(jstate, (Node *) wc->partitionClause);
+ JumbleExpr(jstate, (Node *) wc->orderClause);
+ JumbleExpr(jstate, wc->startOffset);
+ JumbleExpr(jstate, wc->endOffset);
+ }
+ break;
+ case T_CommonTableExpr:
+ {
+ CommonTableExpr *cte = (CommonTableExpr *) node;
+
+ /* we store the string name because RTE_CTE RTEs need it */
+ APP_JUMB_STRING(cte->ctename);
+ APP_JUMB(cte->ctematerialized);
+ JumbleQueryInternal(jstate, castNode(Query, cte->ctequery));
+ }
+ break;
+ case T_SetOperationStmt:
+ {
+ SetOperationStmt *setop = (SetOperationStmt *) node;
+
+ APP_JUMB(setop->op);
+ APP_JUMB(setop->all);
+ JumbleExpr(jstate, setop->larg);
+ JumbleExpr(jstate, setop->rarg);
+ }
+ break;
+ case T_RangeTblFunction:
+ {
+ RangeTblFunction *rtfunc = (RangeTblFunction *) node;
+
+ JumbleExpr(jstate, rtfunc->funcexpr);
+ }
+ break;
+ case T_TableFunc:
+ {
+ TableFunc *tablefunc = (TableFunc *) node;
+
+ JumbleExpr(jstate, tablefunc->docexpr);
+ JumbleExpr(jstate, tablefunc->rowexpr);
+ JumbleExpr(jstate, (Node *) tablefunc->colexprs);
+ }
+ break;
+ case T_TableSampleClause:
+ {
+ TableSampleClause *tsc = (TableSampleClause *) node;
+
+ APP_JUMB(tsc->tsmhandler);
+ JumbleExpr(jstate, (Node *) tsc->args);
+ JumbleExpr(jstate, (Node *) tsc->repeatable);
+ }
+ break;
+ default:
+ /* Only a warning, since we can stumble along anyway */
+ elog(WARNING, "unrecognized node type: %d",
+ (int) nodeTag(node));
+ break;
+ }
+}
+
+/*
+ * Record location of constant within query string of query tree
+ * that is currently being walked.
+ */
+static void
+RecordConstLocation(JumbleState *jstate, int location)
+{
+ /* -1 indicates unknown or undefined location */
+ if (location >= 0)
+ {
+ /* enlarge array if needed */
+ if (jstate->clocations_count >= jstate->clocations_buf_size)
+ {
+ jstate->clocations_buf_size *= 2;
+ jstate->clocations = (LocationLen *)
+ repalloc(jstate->clocations,
+ jstate->clocations_buf_size *
+ sizeof(LocationLen));
+ }
+ jstate->clocations[jstate->clocations_count].location = location;
+ /* initialize lengths to -1 to simplify third-party module usage */
+ jstate->clocations[jstate->clocations_count].length = -1;
+ jstate->clocations_count++;
+ }
+}
#define ANALYZE_H
#include "parser/parse_node.h"
+#include "utils/queryjumble.h"
/* Hook for plugins to get control at end of parse analysis */
typedef void (*post_parse_analyze_hook_type) (ParseState *pstate,
- Query *query);
+ Query *query,
+ JumbleState *jstate);
extern PGDLLIMPORT post_parse_analyze_hook_type post_parse_analyze_hook;
extern PGDLLIMPORT bool check_function_bodies;
extern bool session_auth_is_superuser;
+extern bool compute_query_id;
extern bool log_duration;
extern int log_parameter_max_length;
extern int log_parameter_max_length_on_error;
--- /dev/null
+/*-------------------------------------------------------------------------
+ *
+ * queryjumble.h
+ * Query normalization and fingerprinting.
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/include/utils/queryjumble.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef QUERYJUBLE_H
+#define QUERYJUBLE_H
+
+#include "nodes/parsenodes.h"
+
+#define JUMBLE_SIZE 1024 /* query serialization buffer size */
+
+/*
+ * Struct for tracking locations/lengths of constants during normalization
+ */
+typedef struct LocationLen
+{
+ int location; /* start offset in query text */
+ int length; /* length in bytes, or -1 to ignore */
+} LocationLen;
+
+/*
+ * Working state for computing a query jumble and producing a normalized
+ * query string
+ */
+typedef struct JumbleState
+{
+ /* Jumble of current query tree */
+ unsigned char *jumble;
+
+ /* Number of bytes used in jumble[] */
+ Size jumble_len;
+
+ /* Array of locations of constants that should be removed */
+ LocationLen *clocations;
+
+ /* Allocated length of clocations array */
+ int clocations_buf_size;
+
+ /* Current number of valid entries in clocations array */
+ int clocations_count;
+
+ /* highest Param id we've seen, in order to start normalization correctly */
+ int highest_extern_param_id;
+} JumbleState;
+
+const char *CleanQuerytext(const char *query, int *location, int *len);
+JumbleState *JumbleQuery(Query *query, const char *querytext);
+
+#endif /* QUERYJUMBLE_H */
projects / users / rhaas / postgres.git / commitdiff