From cb413a3129be3f8be32bbb93f592186bceb416d1 Mon Sep 17 00:00:00 2001 From: amitlan Date: Wed, 22 Dec 2021 16:55:17 +0900 Subject: [PATCH v3] Teach AcquireExecutorLocks() to acquire fewer locks in some cases Currently, AcquireExecutorLocks() loops over the range table of a given PlannedStmt and locks all relations found therein, even those that won't actually be scanned during execution due to being eliminated by "initial" pruning that is applied during the initialization of their owning Append or MergeAppend node. This makes AcquireExecutorLocks() itself do the "initial" pruning on nodes that support it and lock only those relations that are contained in the subnodes that survive the pruning. To that end, AcquireExecutorLocks() now loops over a bitmapset of RT indexes, those of the RTEs of "lockable" relations, instead of the whole range table to find such entries. When pruning is possible, the bitmapset is constructed by walking the plan tree to locate nodes that allow "initial" (or "pre-execution") pruning and disregarding relations from subnodes that don't survive the pruning instructions. PlannedStmt gets a bitmapset field to store the RT indexes of lockable relations that is populated when contructing the flat range table in setrefs.c. It is used as is in the absence of any prunable nodes. PlannedStmt also gets a new field that indicates whether any of the nodes of the plan tree contain "initial" (or "pre-execution") pruning steps, which saves the trouble of walking the plan tree only to find whether that's the case. ExecFindInitialMatchingSubPlans() is refactored to allow being called outside a full-fledged executor context. --- src/backend/executor/execParallel.c | 2 + src/backend/executor/execPartition.c | 159 ++++++++++++----- src/backend/executor/nodeAppend.c | 14 +- src/backend/executor/nodeMergeAppend.c | 15 +- src/backend/nodes/copyfuncs.c | 3 + src/backend/nodes/nodeFuncs.c | 116 +++++++++++++ src/backend/nodes/outfuncs.c | 4 + src/backend/nodes/readfuncs.c | 3 + src/backend/optimizer/plan/planner.c | 2 + src/backend/optimizer/plan/setrefs.c | 10 ++ src/backend/partitioning/partprune.c | 46 +++-- src/backend/utils/cache/plancache.c | 229 +++++++++++++++++++++++-- src/include/executor/execPartition.h | 9 +- src/include/nodes/nodeFuncs.h | 3 + src/include/nodes/pathnodes.h | 6 + src/include/nodes/plannodes.h | 11 ++ src/include/partitioning/partprune.h | 2 + 17 files changed, 561 insertions(+), 73 deletions(-) diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c index 5dd8ab7db2..a2979d7602 100644 --- a/src/backend/executor/execParallel.c +++ b/src/backend/executor/execParallel.c @@ -182,8 +182,10 @@ ExecSerializePlan(Plan *plan, EState *estate) pstmt->transientPlan = false; pstmt->dependsOnRole = false; pstmt->parallelModeNeeded = false; + pstmt->usesPreExecPruning = false; pstmt->planTree = plan; pstmt->rtable = estate->es_range_table; + pstmt->relationRTIs = NULL; pstmt->resultRelations = NIL; pstmt->appendRelations = NIL; diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 90ed1485d1..1a0a5814e4 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -24,6 +24,7 @@ #include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/makefuncs.h" +#include "parser/parsetree.h" #include "partitioning/partbounds.h" #include "partitioning/partdesc.h" #include "partitioning/partprune.h" @@ -186,7 +187,8 @@ static void ExecInitPruningContext(PartitionPruneContext *context, List *pruning_steps, PartitionDesc partdesc, PartitionKey partkey, - PlanState *planstate); + PlanState *planstate, + ExprContext *econtext); static void find_matching_subplans_recurse(PartitionPruningData *prunedata, PartitionedRelPruningData *pprune, bool initial_prune, @@ -1514,7 +1516,9 @@ adjust_partition_colnos(List *colnos, ResultRelInfo *leaf_part_rri) * Build the data structure required for calling * ExecFindInitialMatchingSubPlans and ExecFindMatchingSubPlans. * - * 'planstate' is the parent plan node's execution state. + * 'planstate', if not NULL, is the parent plan node's execution state. It + * can be NULL if being called from outside of the executor, in which case + * 'rtable', 'econtext', and 'partdir' must have been provided. * * 'partitionpruneinfo' is a PartitionPruneInfo as generated by * make_partition_pruneinfo. Here we build a PartitionPruneState containing a @@ -1529,18 +1533,19 @@ adjust_partition_colnos(List *colnos, ResultRelInfo *leaf_part_rri) */ PartitionPruneState * ExecCreatePartitionPruneState(PlanState *planstate, - PartitionPruneInfo *partitionpruneinfo) + PartitionPruneInfo *partitionpruneinfo, + bool consider_exec_steps, + List *rtable, ExprContext *econtext, + PartitionDirectory partdir) { - EState *estate = planstate->state; + EState *estate = planstate ? planstate->state : NULL; PartitionPruneState *prunestate; int n_part_hierarchies; ListCell *lc; int i; - /* For data reading, executor always omits detached partitions */ - if (estate->es_partition_directory == NULL) - estate->es_partition_directory = - CreatePartitionDirectory(estate->es_query_cxt, false); + Assert(partdir != NULL && econtext != NULL && + (estate != NULL || rtable != NIL)); n_part_hierarchies = list_length(partitionpruneinfo->prune_infos); Assert(n_part_hierarchies > 0); @@ -1591,19 +1596,34 @@ ExecCreatePartitionPruneState(PlanState *planstate, PartitionedRelPruneInfo *pinfo = lfirst_node(PartitionedRelPruneInfo, lc2); PartitionedRelPruningData *pprune = &prunedata->partrelprunedata[j]; Relation partrel; + bool close_partrel = false; PartitionDesc partdesc; PartitionKey partkey; /* - * We can rely on the copies of the partitioned table's partition - * key and partition descriptor appearing in its relcache entry, - * because that entry will be held open and locked for the - * duration of this executor run. + * We can rely on the copy of the partitioned table's partition + * key from in its relcache entry, because it can't change (or + * get destroyed) as long as the relation is locked. Partition + * descriptor is taken from the PartitionDirectory associated with + * the table that is held open long enough for the descriptor to + * remain valid while it's used to perform the pruning steps. */ - partrel = ExecGetRangeTableRelation(estate, pinfo->rtindex); + if (estate == NULL) + { + RangeTblEntry *rte = rt_fetch(pinfo->rtindex, rtable); + + partrel = table_open(rte->relid, rte->rellockmode); + close_partrel = true; + } + else + partrel = ExecGetRangeTableRelation(estate, pinfo->rtindex); + partkey = RelationGetPartitionKey(partrel); - partdesc = PartitionDirectoryLookup(estate->es_partition_directory, - partrel); + partdesc = PartitionDirectoryLookup(partdir, partrel); + + /* Safe to close partrel, if necessary, keeping the lock taken. */ + if (close_partrel) + table_close(partrel, NoLock); /* * Initialize the subplan_map and subpart_map. @@ -1709,26 +1729,31 @@ ExecCreatePartitionPruneState(PlanState *planstate, { ExecInitPruningContext(&pprune->initial_context, pinfo->initial_pruning_steps, - partdesc, partkey, planstate); + partdesc, partkey, planstate, + econtext); /* Record whether initial pruning is needed at any level */ prunestate->do_initial_prune = true; } pprune->exec_pruning_steps = pinfo->exec_pruning_steps; - if (pinfo->exec_pruning_steps) + if (consider_exec_steps) { - ExecInitPruningContext(&pprune->exec_context, - pinfo->exec_pruning_steps, - partdesc, partkey, planstate); - /* Record whether exec pruning is needed at any level */ - prunestate->do_exec_prune = true; - } + if (pinfo->exec_pruning_steps) + { + ExecInitPruningContext(&pprune->exec_context, + pinfo->exec_pruning_steps, + partdesc, partkey, planstate, + econtext); + /* Record whether exec pruning is needed at any level */ + prunestate->do_exec_prune = true; + } - /* - * Accumulate the IDs of all PARAM_EXEC Params affecting the - * partitioning decisions at this plan node. - */ - prunestate->execparamids = bms_add_members(prunestate->execparamids, - pinfo->execparamids); + /* + * Accumulate the IDs of all PARAM_EXEC Params affecting the + * partitioning decisions at this plan node. + */ + prunestate->execparamids = bms_add_members(prunestate->execparamids, + pinfo->execparamids); + } j++; } @@ -1740,13 +1765,18 @@ ExecCreatePartitionPruneState(PlanState *planstate, /* * Initialize a PartitionPruneContext for the given list of pruning steps. + * + * At least one of 'planstate' or 'econtext' must be passed to be able to + * successfully evaluate any non-Const expressions contained in the + * steps. */ static void ExecInitPruningContext(PartitionPruneContext *context, List *pruning_steps, PartitionDesc partdesc, PartitionKey partkey, - PlanState *planstate) + PlanState *planstate, + ExprContext *econtext) { int n_steps; int partnatts; @@ -1767,6 +1797,7 @@ ExecInitPruningContext(PartitionPruneContext *context, context->ppccontext = CurrentMemoryContext; context->planstate = planstate; + context->exprcontext = econtext; /* Initialize expression state for each expression we need */ context->exprstates = (ExprState **) @@ -1795,8 +1826,13 @@ ExecInitPruningContext(PartitionPruneContext *context, step->step.step_id, keyno); - context->exprstates[stateidx] = - ExecInitExpr(expr, context->planstate); + if (planstate == NULL) + context->exprstates[stateidx] = + ExecInitExprWithParams(expr, + econtext->ecxt_param_list_info); + else + context->exprstates[stateidx] = + ExecInitExpr(expr, context->planstate); } keyno++; } @@ -1818,9 +1854,15 @@ ExecInitPruningContext(PartitionPruneContext *context, * is required. * * 'nsubplans' must be passed as the total number of unpruned subplans. + * + * The RT indexes of unpruned parents are returned in *parentrelids if asked + * for by the caller, in which case 'pruneinfo' must also be passed because + * that is where the RT indexes are to be found. */ Bitmapset * -ExecFindInitialMatchingSubPlans(PartitionPruneState *prunestate, int nsubplans) +ExecFindInitialMatchingSubPlans(PartitionPruneState *prunestate, int nsubplans, + PartitionPruneInfo *pruneinfo, + Bitmapset **parentrelids) { Bitmapset *result = NULL; MemoryContext oldcontext; @@ -1830,11 +1872,14 @@ ExecFindInitialMatchingSubPlans(PartitionPruneState *prunestate, int nsubplans) Assert(prunestate->do_initial_prune); /* - * Switch to a temp context to avoid leaking memory in the executor's - * query-lifespan memory context. + * Switch to a temp context to avoid leaking memory in the longer-term + * memory context. */ oldcontext = MemoryContextSwitchTo(prunestate->prune_context); + if (parentrelids) + *parentrelids = NULL; + /* * For each hierarchy, do the pruning tests, and add nondeletable * subplans' indexes to "result". @@ -1845,14 +1890,42 @@ ExecFindInitialMatchingSubPlans(PartitionPruneState *prunestate, int nsubplans) PartitionedRelPruningData *pprune; prunedata = prunestate->partprunedata[i]; + + /* + * We pass the 1st item belonging to the root table of the hierarchy + * and find_matching_subplans_recurse() takes care of recursing to + * other (lower-level) parents as needed. + */ pprune = &prunedata->partrelprunedata[0]; /* Perform pruning without using PARAM_EXEC Params */ find_matching_subplans_recurse(prunedata, pprune, true, &result); - /* Expression eval may have used space in node's ps_ExprContext too */ + /* + * Collect the RT indexes of surviving parents if the callers asked + * to see them. + */ + if (parentrelids) + { + int j; + List *partrelpruneinfos = list_nth_node(List, + pruneinfo->prune_infos, + i); + + for (j = 0; j < prunedata->num_partrelprunedata; j++) + { + PartitionedRelPruneInfo *pinfo = list_nth_node(PartitionedRelPruneInfo, + partrelpruneinfos, j); + + pprune = &prunedata->partrelprunedata[j]; + if (!bms_is_empty(pprune->present_parts)) + *parentrelids = bms_add_member(*parentrelids, pinfo->rtindex); + } + } + + /* Expression eval may have used space in ExprContext too */ if (pprune->initial_pruning_steps) - ResetExprContext(pprune->initial_context.planstate->ps_ExprContext); + ResetExprContext(pprune->initial_context.exprcontext); } /* Add in any subplans that partition pruning didn't account for */ @@ -1862,9 +1935,12 @@ ExecFindInitialMatchingSubPlans(PartitionPruneState *prunestate, int nsubplans) /* Copy result out of the temp context before we reset it */ result = bms_copy(result); + if (parentrelids) + *parentrelids = bms_copy(*parentrelids); MemoryContextReset(prunestate->prune_context); + /* * If exec-time pruning is required and we pruned subplans above, then we * must re-sequence the subplan indexes so that ExecFindMatchingSubPlans @@ -2018,11 +2094,16 @@ ExecFindMatchingSubPlans(PartitionPruneState *prunestate) prunedata = prunestate->partprunedata[i]; pprune = &prunedata->partrelprunedata[0]; + /* + * We pass the 1st item belonging to the root table of the hierarchy + * and find_matching_subplans_recurse() takes care of recursing to + * other (lower-level) parents as needed. + */ find_matching_subplans_recurse(prunedata, pprune, false, &result); - /* Expression eval may have used space in node's ps_ExprContext too */ + /* Expression eval may have used space in ExprContext too */ if (pprune->exec_pruning_steps) - ResetExprContext(pprune->exec_context.planstate->ps_ExprContext); + ResetExprContext(pprune->exec_context.exprcontext); } /* Add in any subplans that partition pruning didn't account for */ diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index 7937f1c88f..51aac946fa 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -62,6 +62,7 @@ #include "executor/execPartition.h" #include "executor/nodeAppend.h" #include "miscadmin.h" +#include "partitioning/partdesc.h" #include "pgstat.h" #include "storage/latch.h" @@ -141,9 +142,16 @@ ExecInitAppend(Append *node, EState *estate, int eflags) /* We may need an expression context to evaluate partition exprs */ ExecAssignExprContext(estate, &appendstate->ps); + /* For data reading, executor always omits detached partitions */ + if (estate->es_partition_directory == NULL) + estate->es_partition_directory = + CreatePartitionDirectory(estate->es_query_cxt, false); + /* Create the working data structure for pruning. */ prunestate = ExecCreatePartitionPruneState(&appendstate->ps, - node->part_prune_info); + node->part_prune_info, true, + NIL, appendstate->ps.ps_ExprContext, + estate->es_partition_directory); appendstate->as_prune_state = prunestate; /* Perform an initial partition prune, if required. */ @@ -151,7 +159,9 @@ ExecInitAppend(Append *node, EState *estate, int eflags) { /* Determine which subplans survive initial pruning */ validsubplans = ExecFindInitialMatchingSubPlans(prunestate, - list_length(node->appendplans)); + list_length(node->appendplans), + node->part_prune_info, + NULL); nplans = bms_num_members(validsubplans); } diff --git a/src/backend/executor/nodeMergeAppend.c b/src/backend/executor/nodeMergeAppend.c index 418f89dea8..7d1185ec9d 100644 --- a/src/backend/executor/nodeMergeAppend.c +++ b/src/backend/executor/nodeMergeAppend.c @@ -43,6 +43,7 @@ #include "executor/nodeMergeAppend.h" #include "lib/binaryheap.h" #include "miscadmin.h" +#include "partitioning/partdesc.h" /* * We have one slot for each item in the heap array. We use SlotNumber @@ -89,8 +90,16 @@ ExecInitMergeAppend(MergeAppend *node, EState *estate, int eflags) /* We may need an expression context to evaluate partition exprs */ ExecAssignExprContext(estate, &mergestate->ps); + /* For data reading, executor always omits detached partitions */ + if (estate->es_partition_directory == NULL) + estate->es_partition_directory = + CreatePartitionDirectory(estate->es_query_cxt, false); + + /* Create the working data structure for pruning. */ prunestate = ExecCreatePartitionPruneState(&mergestate->ps, - node->part_prune_info); + node->part_prune_info, true, + NIL, mergestate->ps.ps_ExprContext, + estate->es_partition_directory); mergestate->ms_prune_state = prunestate; /* Perform an initial partition prune, if required. */ @@ -98,7 +107,9 @@ ExecInitMergeAppend(MergeAppend *node, EState *estate, int eflags) { /* Determine which subplans survive initial pruning */ validsubplans = ExecFindInitialMatchingSubPlans(prunestate, - list_length(node->mergeplans)); + list_length(node->mergeplans), + node->part_prune_info, + NULL); nplans = bms_num_members(validsubplans); } diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index b105c26381..4b539f792b 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -94,9 +94,11 @@ _copyPlannedStmt(const PlannedStmt *from) COPY_SCALAR_FIELD(transientPlan); COPY_SCALAR_FIELD(dependsOnRole); COPY_SCALAR_FIELD(parallelModeNeeded); + COPY_SCALAR_FIELD(usesPreExecPruning); COPY_SCALAR_FIELD(jitFlags); COPY_NODE_FIELD(planTree); COPY_NODE_FIELD(rtable); + COPY_BITMAPSET_FIELD(relationRTIs); COPY_NODE_FIELD(resultRelations); COPY_NODE_FIELD(appendRelations); COPY_NODE_FIELD(subplans); @@ -1278,6 +1280,7 @@ _copyPartitionPruneInfo(const PartitionPruneInfo *from) PartitionPruneInfo *newnode = makeNode(PartitionPruneInfo); COPY_NODE_FIELD(prune_infos); + COPY_SCALAR_FIELD(contains_init_steps); COPY_BITMAPSET_FIELD(other_subplans); return newnode; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index acc17da717..d2de60711d 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -31,6 +31,10 @@ static bool planstate_walk_subplans(List *plans, bool (*walker) (), void *context); static bool planstate_walk_members(PlanState **planstates, int nplans, bool (*walker) (), void *context); +static bool plan_walk_subplans(List *plans, + bool (*walker) (), + void *context); +static bool plan_walk_members(List *plans, bool (*walker) (), void *context); /* @@ -4147,3 +4151,115 @@ planstate_walk_members(PlanState **planstates, int nplans, return false; } + +/* + * plan_tree_walker --- walk plantrees + * + * The walker has already visited the current node, and so we need only + * recurse into any sub-nodes it has. + */ +bool +plan_tree_walker(Plan *plan, + bool (*walker) (), + void *context) +{ + /* Guard against stack overflow due to overly complex plan trees */ + check_stack_depth(); + + /* initPlan-s */ + if (plan_walk_subplans(plan->initPlan, walker, context)) + return true; + + /* lefttree */ + if (outerPlan(plan)) + { + if (walker(outerPlan(plan), context)) + return true; + } + + /* righttree */ + if (innerPlan(plan)) + { + if (walker(innerPlan(plan), context)) + return true; + } + + /* special child plans */ + switch (nodeTag(plan)) + { + case T_Append: + if (plan_walk_members(((Append *) plan)->appendplans, + walker, context)) + return true; + break; + case T_MergeAppend: + if (plan_walk_members(((MergeAppend *) plan)->mergeplans, + walker, context)) + return true; + break; + case T_BitmapAnd: + if (plan_walk_members(((BitmapAnd *) plan)->bitmapplans, + walker, context)) + return true; + break; + case T_BitmapOr: + if (plan_walk_members(((BitmapOr *) plan)->bitmapplans, + walker, context)) + return true; + break; + case T_CustomScan: + if (plan_walk_members(((CustomScan *) plan)->custom_plans, + walker, context)) + return true; + break; + case T_SubqueryScan: + if (walker(((SubqueryScan *) plan)->subplan, context)) + return true; + break; + default: + break; + } + + return false; +} + +/* + * Walk a list of SubPlans (or initPlans, which also use SubPlan nodes). + */ +static bool +plan_walk_subplans(List *plans, + bool (*walker) (), + void *context) +{ + ListCell *lc; + PlannedStmt *plannedstmt = (PlannedStmt *) context; + + foreach(lc, plans) + { + SubPlan *sp = lfirst_node(SubPlan, lc); + Plan *p = list_nth(plannedstmt->subplans, sp->plan_id - 1); + + if (walker(p, context)) + return true; + } + + return false; +} + +/* + * Walk the constituent plans of a ModifyTable, Append, MergeAppend, + * BitmapAnd, or BitmapOr node. + */ +static bool +plan_walk_members(List *plans, bool (*walker) (), void *context) +{ + ListCell *lc; + + foreach(lc, plans) + { + if (walker(lfirst(lc), context)) + return true; + } + + return false; +} diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d28cea1567..994971d0cb 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -312,9 +312,11 @@ _outPlannedStmt(StringInfo str, const PlannedStmt *node) WRITE_BOOL_FIELD(transientPlan); WRITE_BOOL_FIELD(dependsOnRole); WRITE_BOOL_FIELD(parallelModeNeeded); + WRITE_BOOL_FIELD(usesPreExecPruning); WRITE_INT_FIELD(jitFlags); WRITE_NODE_FIELD(planTree); WRITE_NODE_FIELD(rtable); + WRITE_BITMAPSET_FIELD(relationRTIs); WRITE_NODE_FIELD(resultRelations); WRITE_NODE_FIELD(appendRelations); WRITE_NODE_FIELD(subplans); @@ -1004,6 +1006,7 @@ _outPartitionPruneInfo(StringInfo str, const PartitionPruneInfo *node) WRITE_NODE_TYPE("PARTITIONPRUNEINFO"); WRITE_NODE_FIELD(prune_infos); + WRITE_BOOL_FIELD(contains_init_steps); WRITE_BITMAPSET_FIELD(other_subplans); } @@ -2274,6 +2277,7 @@ _outPlannerGlobal(StringInfo str, const PlannerGlobal *node) WRITE_NODE_FIELD(subplans); WRITE_BITMAPSET_FIELD(rewindPlanIDs); WRITE_NODE_FIELD(finalrtable); + WRITE_BITMAPSET_FIELD(relationRTIs); WRITE_NODE_FIELD(finalrowmarks); WRITE_NODE_FIELD(resultRelations); WRITE_NODE_FIELD(appendRelations); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 3f68f7c18d..8b3caeef03 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1585,9 +1585,11 @@ _readPlannedStmt(void) READ_BOOL_FIELD(transientPlan); READ_BOOL_FIELD(dependsOnRole); READ_BOOL_FIELD(parallelModeNeeded); + READ_BOOL_FIELD(usesPreExecPruning); READ_INT_FIELD(jitFlags); READ_NODE_FIELD(planTree); READ_NODE_FIELD(rtable); + READ_BITMAPSET_FIELD(relationRTIs); READ_NODE_FIELD(resultRelations); READ_NODE_FIELD(appendRelations); READ_NODE_FIELD(subplans); @@ -2534,6 +2536,7 @@ _readPartitionPruneInfo(void) READ_LOCALS(PartitionPruneInfo); READ_NODE_FIELD(prune_infos); + READ_BOOL_FIELD(contains_init_steps); READ_BITMAPSET_FIELD(other_subplans); READ_DONE(); diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index bd09f85aea..3f35f8f892 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -517,8 +517,10 @@ standard_planner(Query *parse, const char *query_string, int cursorOptions, result->transientPlan = glob->transientPlan; result->dependsOnRole = glob->dependsOnRole; result->parallelModeNeeded = glob->parallelModeNeeded; + result->usesPreExecPruning = glob->usesPreExecPruning; result->planTree = top_plan; result->rtable = glob->finalrtable; + result->relationRTIs = glob->relationRTIs; result->resultRelations = glob->resultRelations; result->appendRelations = glob->appendRelations; result->subplans = glob->subplans; diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index e44ae971b4..d34a7eb621 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -483,6 +483,7 @@ static void add_rte_to_flat_rtable(PlannerGlobal *glob, RangeTblEntry *rte) { RangeTblEntry *newrte; + Index rti = list_length(glob->finalrtable) + 1; /* flat copy to duplicate all the scalar fields */ newrte = (RangeTblEntry *) palloc(sizeof(RangeTblEntry)); @@ -517,7 +518,10 @@ add_rte_to_flat_rtable(PlannerGlobal *glob, RangeTblEntry *rte) * but it would probably cost more cycles than it would save. */ if (newrte->rtekind == RTE_RELATION) + { + glob->relationRTIs = bms_add_member(glob->relationRTIs, rti); glob->relationOids = lappend_oid(glob->relationOids, newrte->relid); + } } /* @@ -1540,6 +1544,9 @@ set_append_references(PlannerInfo *root, pinfo->rtindex += rtoffset; } } + + if (aplan->part_prune_info->contains_init_steps) + root->glob->usesPreExecPruning = true; } /* We don't need to recurse to lefttree or righttree ... */ @@ -1604,6 +1611,9 @@ set_mergeappend_references(PlannerInfo *root, pinfo->rtindex += rtoffset; } } + + if (mplan->part_prune_info->contains_init_steps) + root->glob->usesPreExecPruning = true; } /* We don't need to recurse to lefttree or righttree ... */ diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c index 1bc00826c1..3e3c6c78df 100644 --- a/src/backend/partitioning/partprune.c +++ b/src/backend/partitioning/partprune.c @@ -144,7 +144,8 @@ static List *make_partitionedrel_pruneinfo(PlannerInfo *root, List *prunequal, Bitmapset *partrelids, int *relid_subplan_map, - Bitmapset **matchedsubplans); + Bitmapset **matchedsubplans, + bool *contains_init_steps); static void gen_partprune_steps(RelOptInfo *rel, List *clauses, PartClauseTarget target, GeneratePruningStepsContext *context); @@ -230,6 +231,7 @@ make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, int *relid_subplan_map; ListCell *lc; int i; + bool contains_init_steps = false; /* * Scan the subpaths to see which ones are scans of partition child @@ -309,12 +311,14 @@ make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, Bitmapset *partrelids = (Bitmapset *) lfirst(lc); List *pinfolist; Bitmapset *matchedsubplans = NULL; + bool partrel_contains_init_steps; pinfolist = make_partitionedrel_pruneinfo(root, parentrel, prunequal, partrelids, relid_subplan_map, - &matchedsubplans); + &matchedsubplans, + &partrel_contains_init_steps); /* When pruning is possible, record the matched subplans */ if (pinfolist != NIL) @@ -323,6 +327,8 @@ make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, allmatchedsubplans = bms_join(matchedsubplans, allmatchedsubplans); } + if (!contains_init_steps) + contains_init_steps = partrel_contains_init_steps; } pfree(relid_subplan_map); @@ -337,6 +343,7 @@ make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, /* Else build the result data structure */ pruneinfo = makeNode(PartitionPruneInfo); pruneinfo->prune_infos = prunerelinfos; + pruneinfo->contains_init_steps = contains_init_steps; /* * Some subplans may not belong to any of the identified partitioned rels. @@ -435,13 +442,17 @@ add_part_relids(List *allpartrelids, Bitmapset *partrelids) * If we cannot find any useful run-time pruning steps, return NIL. * However, on success, each rel identified in partrelids will have * an element in the result list, even if some of them are useless. + * *contains_init_steps and are set to indicate that the returned + * PartitionedRelPruneInfos contains pruning steps that can be performed + * before execution begins. */ static List * make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, List *prunequal, Bitmapset *partrelids, int *relid_subplan_map, - Bitmapset **matchedsubplans) + Bitmapset **matchedsubplans, + bool *contains_init_steps) { RelOptInfo *targetpart = NULL; List *pinfolist = NIL; @@ -452,6 +463,9 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, int rti; int i; + /* Will find out below. */ + *contains_init_steps = false; + /* * Examine each partitioned rel, constructing a temporary array to map * from planner relids to index of the partitioned rel, and building a @@ -539,6 +553,9 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, * executor per-scan pruning steps. This first pass creates startup * pruning steps and detects whether there's any possibly-useful quals * that would require per-scan pruning. + * + * In the first pass, we note whether the 2nd pass is necessary by + * by noting the presence of EXEC parameters. */ gen_partprune_steps(subpart, partprunequal, PARTTARGET_INITIAL, &context); @@ -613,6 +630,9 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, pinfo->execparamids = execparamids; /* Remaining fields will be filled in the next loop */ + if (!*contains_init_steps) + *contains_init_steps = (initial_pruning_steps != NIL); + pinfolist = lappend(pinfolist, pinfo); } @@ -798,6 +818,7 @@ prune_append_rel_partitions(RelOptInfo *rel) /* These are not valid when being called from the planner */ context.planstate = NULL; + context.exprcontext = NULL; context.exprstates = NULL; /* Actual pruning happens here. */ @@ -808,8 +829,8 @@ prune_append_rel_partitions(RelOptInfo *rel) * get_matching_partitions * Determine partitions that survive partition pruning * - * Note: context->planstate must be set to a valid PlanState when the - * pruning_steps were generated with a target other than PARTTARGET_PLANNER. + * Note: context->exprcontext must be valid when the pruning_steps were + * generated with a target other than PARTTARGET_PLANNER. * * Returns a Bitmapset of the RelOptInfo->part_rels indexes of the surviving * partitions. @@ -3654,7 +3675,7 @@ match_boolean_partition_clause(Oid partopfamily, Expr *clause, Expr *partkey, * exprstate array. * * Note that the evaluated result may be in the per-tuple memory context of - * context->planstate->ps_ExprContext, and we may have leaked other memory + * context->exprcontext, and we may have leaked other memory * there too. This memory must be recovered by resetting that ExprContext * after we're done with the pruning operation (see execPartition.c). */ @@ -3677,13 +3698,18 @@ partkey_datum_from_expr(PartitionPruneContext *context, ExprContext *ectx; /* - * We should never see a non-Const in a step unless we're running in - * the executor. + * We should never see a non-Const in a step unless the caller has + * passed a valid ExprContext. + * + * When context->planstate is valid, context->exprcontext is same + * as context->planstate->ps_ExprContext. */ - Assert(context->planstate != NULL); + Assert(context->planstate != NULL || context->exprcontext != NULL); + Assert(context->planstate == NULL || + (context->exprcontext == context->planstate->ps_ExprContext)); exprstate = context->exprstates[stateidx]; - ectx = context->planstate->ps_ExprContext; + ectx = context->exprcontext; *value = ExecEvalExprSwitchContext(exprstate, ectx, isnull); } } diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c index 4a9055e6bb..6c4c6f0d95 100644 --- a/src/backend/utils/cache/plancache.c +++ b/src/backend/utils/cache/plancache.c @@ -58,12 +58,14 @@ #include "access/transam.h" #include "catalog/namespace.h" +#include "executor/execPartition.h" #include "executor/executor.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" #include "parser/analyze.h" #include "parser/parsetree.h" +#include "partitioning/partdesc.h" #include "storage/lmgr.h" #include "tcop/pquery.h" #include "tcop/utility.h" @@ -99,14 +101,25 @@ static dlist_head cached_expression_list = DLIST_STATIC_INIT(cached_expression_l static void ReleaseGenericPlan(CachedPlanSource *plansource); static List *RevalidateCachedQuery(CachedPlanSource *plansource, QueryEnvironment *queryEnv); -static bool CheckCachedPlan(CachedPlanSource *plansource); +static bool CheckCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams); static CachedPlan *BuildCachedPlan(CachedPlanSource *plansource, List *qlist, ParamListInfo boundParams, QueryEnvironment *queryEnv); static bool choose_custom_plan(CachedPlanSource *plansource, ParamListInfo boundParams); static double cached_plan_cost(CachedPlan *plan, bool include_planner); static Query *QueryListGetPrimaryStmt(List *stmts); -static void AcquireExecutorLocks(List *stmt_list, bool acquire); +static void AcquireExecutorLocks(List *stmt_list, bool acquire, + ParamListInfo boundParams); +struct GetLockableRelations_context +{ + PlannedStmt *plannedstmt; + Bitmapset *relations; + ParamListInfo params; +}; +static Bitmapset *GetLockableRelations(PlannedStmt *plannedstmt, + ParamListInfo boundParams); +static bool GetLockableRelations_worker(Plan *plan, + struct GetLockableRelations_context *context); static void AcquirePlannerLocks(List *stmt_list, bool acquire); static void ScanQueryForLocks(Query *parsetree, bool acquire); static bool ScanQueryWalker(Node *node, bool *acquire); @@ -792,7 +805,7 @@ RevalidateCachedQuery(CachedPlanSource *plansource, * (We must do this for the "true" result to be race-condition-free.) */ static bool -CheckCachedPlan(CachedPlanSource *plansource) +CheckCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams) { CachedPlan *plan = plansource->gplan; @@ -826,7 +839,7 @@ CheckCachedPlan(CachedPlanSource *plansource) */ Assert(plan->refcount > 0); - AcquireExecutorLocks(plan->stmt_list, true); + AcquireExecutorLocks(plan->stmt_list, true, boundParams); /* * If plan was transient, check to see if TransactionXmin has @@ -848,7 +861,7 @@ CheckCachedPlan(CachedPlanSource *plansource) } /* Oops, the race case happened. Release useless locks. */ - AcquireExecutorLocks(plan->stmt_list, false); + AcquireExecutorLocks(plan->stmt_list, false, boundParams); } /* @@ -1160,7 +1173,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, if (!customplan) { - if (CheckCachedPlan(plansource)) + if (CheckCachedPlan(plansource, boundParams)) { /* We want a generic plan, and we already have a valid one */ plan = plansource->gplan; @@ -1366,7 +1379,6 @@ CachedPlanAllowsSimpleValidityCheck(CachedPlanSource *plansource, foreach(lc, plan->stmt_list) { PlannedStmt *plannedstmt = lfirst_node(PlannedStmt, lc); - ListCell *lc2; if (plannedstmt->commandType == CMD_UTILITY) return false; @@ -1375,13 +1387,8 @@ CachedPlanAllowsSimpleValidityCheck(CachedPlanSource *plansource, * We have to grovel through the rtable because it's likely to contain * an RTE_RESULT relation, rather than being totally empty. */ - foreach(lc2, plannedstmt->rtable) - { - RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc2); - - if (rte->rtekind == RTE_RELATION) - return false; - } + if (!bms_is_empty(plannedstmt->relationRTIs)) + return false; } /* @@ -1740,14 +1747,15 @@ QueryListGetPrimaryStmt(List *stmts) * or release them if acquire is false. */ static void -AcquireExecutorLocks(List *stmt_list, bool acquire) +AcquireExecutorLocks(List *stmt_list, bool acquire, ParamListInfo boundParams) { ListCell *lc1; foreach(lc1, stmt_list) { PlannedStmt *plannedstmt = lfirst_node(PlannedStmt, lc1); - ListCell *lc2; + Bitmapset *relations; + int rti; if (plannedstmt->commandType == CMD_UTILITY) { @@ -1765,9 +1773,22 @@ AcquireExecutorLocks(List *stmt_list, bool acquire) continue; } - foreach(lc2, plannedstmt->rtable) + /* + * Fetch the RT indexes of only the relations that will be actually + * scanned when the plan is executed. This skips over scan nodes + * appearing as child subnodes of any Append/MergeAppend nodes present + * in the plan tree. It does so by performing + * ExecFindInitialMatchingSubPlans() to run any pruning steps + * contained in those nodes that can be safely run at this point, using + * 'boundParams' to evaluate any EXTERN parameters contained in the + * steps. + */ + relations = GetLockableRelations(plannedstmt, boundParams); + + rti = -1; + while ((rti = bms_next_member(relations, rti)) >= 0) { - RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc2); + RangeTblEntry *rte = rt_fetch(rti, plannedstmt->rtable); if (rte->rtekind != RTE_RELATION) continue; @@ -1786,6 +1807,178 @@ AcquireExecutorLocks(List *stmt_list, bool acquire) } } +/* + * GetLockableRelations + * Returns set of RT indexes of relations that must be locked by + * AcquireExecutorLocks() + */ +static Bitmapset * +GetLockableRelations(PlannedStmt *plannedstmt, ParamListInfo boundParams) +{ + ListCell *lc; + struct GetLockableRelations_context context; + + /* None of the relation scanning nodes are prunable here. */ + if (!plannedstmt->usesPreExecPruning) + return plannedstmt->relationRTIs; + + /* + * Look for prunable nodes in the main plan tree, followed by those in + * subplans. + */ + context.plannedstmt = plannedstmt; + context.params = boundParams; + context.relations = NULL; + + (void) GetLockableRelations_worker(plannedstmt->planTree, &context); + + foreach(lc, plannedstmt->subplans) + { + Plan *subplan = lfirst(lc); + + (void) GetLockableRelations_worker(subplan, &context); + } + + return context.relations; +} + +/* + * GetLockableRelations_worker + * Adds RT indexes of relations to be scanned by plan to + * context->relations + * + * For plan node types that support pruning, this only adds child plan + * subnodes that satisfy the "initial" pruning steps. + */ +static bool +GetLockableRelations_worker(Plan *plan, + struct GetLockableRelations_context *context) +{ + if (plan == NULL) + return false; + + switch(nodeTag(plan)) + { + /* Nodes scanning a relation or relations. */ + case T_SeqScan: + case T_SampleScan: + case T_IndexScan: + case T_IndexOnlyScan: + case T_BitmapHeapScan: + case T_TidScan: + case T_TidRangeScan: + context->relations = bms_add_member(context->relations, + ((Scan *) plan)->scanrelid); + return false; + case T_ForeignScan: + context->relations = bms_add_members(context->relations, + ((ForeignScan *) plan)->fs_relids); + return false; + case T_CustomScan: + context->relations = bms_add_members(context->relations, + ((CustomScan *) plan)->custom_relids); + return false; + + /* Nodes containing prunable subnodes. */ + case T_Append: + case T_MergeAppend: + { + PartitionPruneInfo *pruneinfo; + + if (IsA(plan, Append)) + pruneinfo = ((Append *) plan)->part_prune_info; + else + pruneinfo = ((MergeAppend *) plan)->part_prune_info; + + if (pruneinfo && pruneinfo->contains_init_steps) + { + List *rtable = context->plannedstmt->rtable; + ParamListInfo params = context->params; + List *subplans; + Bitmapset *validsubplans; + Bitmapset *parentrelids; + int i; + ExprContext *econtext; + PartitionDirectory pdir; + MemoryContext oldcontext, + tmpcontext; + PartitionPruneState *prunestate; + + if (IsA(plan, Append)) + subplans = ((Append *) plan)->appendplans; + else + subplans = ((MergeAppend *) plan)->mergeplans; + + /* + * A temporary context to allocate stuff needded to run + * the pruning steps. + */ + tmpcontext = AllocSetContextCreate(CurrentMemoryContext, + "initial pruning working data", + ALLOCSET_DEFAULT_SIZES); + oldcontext = MemoryContextSwitchTo(tmpcontext); + + /* An ExprContext to evaluate expressions. */ + econtext = CreateStandaloneExprContext(); + econtext->ecxt_param_list_info = params; + + /* + * PartitionDirectory, to look up partition descriptors + * Omits detached partitions, just like in the executor + * proper. + */ + pdir = CreatePartitionDirectory(CurrentMemoryContext, false); + prunestate = ExecCreatePartitionPruneState(NULL, + pruneinfo, false, + rtable, econtext, + pdir); + MemoryContextSwitchTo(oldcontext); + + /* Do the "initial" pruning. */ + validsubplans = + ExecFindInitialMatchingSubPlans(prunestate, + list_length(subplans), + pruneinfo, + &parentrelids); + + FreeExprContext(econtext, true); + DestroyPartitionDirectory(pdir); + MemoryContextDelete(tmpcontext); + + /* All relevant parents must be locked. */ + Assert(bms_num_members(parentrelids) > 0); + context->relations = bms_add_members(context->relations, + parentrelids); + + /* And all leaf partitions that will be scanned. */ + i = -1; + while ((i = bms_next_member(validsubplans, i)) >= 0) + { + Plan *subplan = list_nth(subplans, i); + + GetLockableRelations_worker(subplan, context); + } + + return false; + } + else + { + /* + * plan_tree_walker() will take care of walking *all* of + * the node's child subplans to collect their relids. + */ + } + } + break; + + default: + break; + } + + return plan_tree_walker(plan, GetLockableRelations_worker, + (void *) context); +} + /* * AcquirePlannerLocks: acquire locks needed for planning of a querytree list; * or release them if acquire is false. diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 603d8becc4..7b77c8d20e 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -120,9 +120,14 @@ extern ResultRelInfo *ExecFindPartition(ModifyTableState *mtstate, extern void ExecCleanupTupleRouting(ModifyTableState *mtstate, PartitionTupleRouting *proute); extern PartitionPruneState *ExecCreatePartitionPruneState(PlanState *planstate, - PartitionPruneInfo *partitionpruneinfo); + PartitionPruneInfo *partitionpruneinfo, + bool consider_exec_steps, + List *rtable, ExprContext *econtext, + PartitionDirectory partdir); extern Bitmapset *ExecFindMatchingSubPlans(PartitionPruneState *prunestate); extern Bitmapset *ExecFindInitialMatchingSubPlans(PartitionPruneState *prunestate, - int nsubplans); + int nsubplans, + PartitionPruneInfo *pruneinfo, + Bitmapset **parentrelids); #endif /* EXECPARTITION_H */ diff --git a/src/include/nodes/nodeFuncs.h b/src/include/nodes/nodeFuncs.h index 93c60bde66..fca107ad65 100644 --- a/src/include/nodes/nodeFuncs.h +++ b/src/include/nodes/nodeFuncs.h @@ -158,5 +158,8 @@ extern bool raw_expression_tree_walker(Node *node, bool (*walker) (), struct PlanState; extern bool planstate_tree_walker(struct PlanState *planstate, bool (*walker) (), void *context); +struct Plan; +extern bool plan_tree_walker(struct Plan *plan, bool (*walker) (), + void *context); #endif /* NODEFUNCS_H */ diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 1f3845b3fe..ffde93ef13 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -101,6 +101,9 @@ typedef struct PlannerGlobal List *finalrtable; /* "flat" rangetable for executor */ + Bitmapset *relationRTIs; /* Indexes of RTE_RELATION entries in range + * table */ + List *finalrowmarks; /* "flat" list of PlanRowMarks */ List *resultRelations; /* "flat" list of integer RT indexes */ @@ -129,6 +132,9 @@ typedef struct PlannerGlobal char maxParallelHazard; /* worst PROPARALLEL hazard level */ + bool usesPreExecPruning; /* Do some Plan nodes use pre-execution + * partition pruning */ + PartitionDirectory partition_directory; /* partition descriptors */ } PlannerGlobal; diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 0b518ce6b2..bdb72f7cbf 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -59,12 +59,18 @@ typedef struct PlannedStmt bool parallelModeNeeded; /* parallel mode required to execute? */ + bool usesPreExecPruning; /* Do some Plan nodes use pre-execution + * partition pruning */ + int jitFlags; /* which forms of JIT should be performed */ struct Plan *planTree; /* tree of Plan nodes */ List *rtable; /* list of RangeTblEntry nodes */ + Bitmapset *relationRTIs; /* Indexes of RTE_RELATION entries in range + * table */ + /* rtable indexes of target relations for INSERT/UPDATE/DELETE */ List *resultRelations; /* integer list of RT indexes, or NIL */ @@ -1172,6 +1178,10 @@ typedef struct PlanRowMark * prune_infos List of Lists containing PartitionedRelPruneInfo nodes, * one sublist per run-time-prunable partition hierarchy * appearing in the parent plan node's subplans. + * + * contains_init_steps Does any of the PartitionedRelPruneInfos in + * prune_infos have its initial_pruning_steps set? + * * other_subplans Indexes of any subplans that are not accounted for * by any of the PartitionedRelPruneInfo nodes in * "prune_infos". These subplans must not be pruned. @@ -1180,6 +1190,7 @@ typedef struct PartitionPruneInfo { NodeTag type; List *prune_infos; + bool contains_init_steps; Bitmapset *other_subplans; } PartitionPruneInfo; diff --git a/src/include/partitioning/partprune.h b/src/include/partitioning/partprune.h index ee11b6feae..90684efa25 100644 --- a/src/include/partitioning/partprune.h +++ b/src/include/partitioning/partprune.h @@ -41,6 +41,7 @@ struct RelOptInfo; * subsidiary data, such as the FmgrInfos. * planstate Points to the parent plan node's PlanState when called * during execution; NULL when called from the planner. + * exprcontext ExprContext to use when evaluating pruning expressions * exprstates Array of ExprStates, indexed as per PruneCxtStateIdx; one * for each partition key in each pruning step. Allocated if * planstate is non-NULL, otherwise NULL. @@ -56,6 +57,7 @@ typedef struct PartitionPruneContext FmgrInfo *stepcmpfuncs; MemoryContext ppccontext; PlanState *planstate; + ExprContext *exprcontext; ExprState **exprstates; } PartitionPruneContext; -- 2.24.1