From 71bb618f69872623366ca85ff4799c99c7ca9e1f Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 21 Mar 2018 17:34:17 -0400 Subject: [PATCH] Refactor partitonwise aggregate signalling. --- src/backend/optimizer/plan/planner.c | 215 ++++++++++++++--------------------- src/include/nodes/relation.h | 26 ++++- 2 files changed, 105 insertions(+), 136 deletions(-) diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index f2b1a8bf39..e3d4dcaae7 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -107,14 +107,10 @@ typedef struct * GROUPING_CAN_PARTIAL_AGG should be set if the aggregation is of a type * for which we support partial aggregation (not, for example, grouping sets). * It says nothing about parallel-safety or the availability of suitable paths. - * - * GROUPING_CAN_PARTITIONWISE_AGG should be set if it's possible to perform - * partitionwise grouping and/or aggregation. */ #define GROUPING_CAN_USE_SORT 0x0001 #define GROUPING_CAN_USE_HASH 0x0002 #define GROUPING_CAN_PARTIAL_AGG 0x0004 -#define GROUPING_CAN_PARTITIONWISE_AGG 0x0008 /* * Data specific to grouping sets @@ -237,7 +233,8 @@ static RelOptInfo *create_partial_grouping_paths(PlannerInfo *root, RelOptInfo *grouped_rel, RelOptInfo *input_rel, grouping_sets_data *gd, - GroupPathExtraData *extra); + GroupPathExtraData *extra, + bool force_rel_creation); static void gather_grouping_paths(PlannerInfo *root, RelOptInfo *rel); static bool can_partial_agg(PlannerInfo *root, const AggClauseCosts *agg_costs); @@ -246,18 +243,13 @@ static void apply_scanjoin_target_to_paths(PlannerInfo *root, PathTarget *scanjoin_target, bool scanjoin_target_parallel_safe, bool modify_in_place); -static bool can_partitionwise_grouping(PlannerInfo *root, - RelOptInfo *input_rel, - RelOptInfo *grouped_rel, - GroupPathExtraData *extra, - grouping_sets_data *gd, - bool *perform_partial_partitionwise_aggregation); static void create_partitionwise_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, RelOptInfo *grouped_rel, RelOptInfo *partially_grouped_rel, const AggClauseCosts *agg_costs, grouping_sets_data *gd, + PartitionwiseAggregateType patype, GroupPathExtraData *extra); static bool group_by_has_partkey(RelOptInfo *input_rel, RelOptInfo *grouped_rel, @@ -3742,16 +3734,17 @@ create_grouping_paths(PlannerInfo *root, extra.havingQual = parse->havingQual; extra.targetList = parse->targetList; extra.partial_costs_set = false; - extra.is_partial_aggregation = false; /* - * Check whether we can perform partitionwise grouping and/or - * aggregation. + * Determine whether partitionwise aggregation is in theory possible. + * It can be disabled by the user, and for now, we don't try to + * support grouping sets. create_ordinary_grouping_paths() will check + * additional conditions, such as whether input_rel is partitioned. */ - if (can_partitionwise_grouping(root, input_rel, grouped_rel, &extra, - gd, - &extra.perform_partial_partitionwise_aggregation)) - extra.flags |= GROUPING_CAN_PARTITIONWISE_AGG; + if (enable_partitionwise_aggregate && !parse->groupingSets) + extra.patype = PARTITIONWISE_AGGREGATE_FULL; + else + extra.patype = PARTITIONWISE_AGGREGATE_NONE; create_ordinary_grouping_paths(root, input_rel, grouped_rel, agg_costs, gd, &extra, @@ -3923,6 +3916,38 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, Path *cheapest_path = input_rel->cheapest_total_path; RelOptInfo *partially_grouped_rel = NULL; double dNumGroups; + PartitionwiseAggregateType patype = PARTITIONWISE_AGGREGATE_NONE; + + /* + * If this is the topmost grouping relation or if the parent relation is + * doing some form of partitionwise aggregation, then we may be able to do + * it at this level also. However, if the input relation is not + * partitioned, partition-wise aggregate is impossible, and if it is dummy, + * partition-wise aggregate is pointless. + */ + if (extra->patype != PARTITIONWISE_AGGREGATE_NONE && + input_rel->part_scheme && input_rel->part_rels && + !IS_DUMMY_REL(input_rel)) + { + /* + * If this is the topmost relation or if the parent relation is doing + * full partitionwise aggregation, then we can do full partitionwise + * aggregation provided that the GROUP BY clause contains all of the + * partitioning columns at this level. Otherwise, we can do at most + * partial partitionwise aggregation. But if partial aggregation is + * not supported in general then we can't use it for partitionwise + * aggregation either. + */ + if (extra->patype == PARTITIONWISE_AGGREGATE_FULL && + group_by_has_partkey(input_rel, grouped_rel, + extra->targetList, + root->parse->groupClause)) + patype = PARTITIONWISE_AGGREGATE_FULL; + else if ((extra->flags & GROUPING_CAN_PARTIAL_AGG) != 0) + patype = PARTITIONWISE_AGGREGATE_PARTIAL; + else + patype = PARTITIONWISE_AGGREGATE_NONE; + } /* * Before generating paths for grouped_rel, we first generate any possible @@ -3930,12 +3955,24 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, * parallel and non-parallel approaches to grouping. */ if ((extra->flags & GROUPING_CAN_PARTIAL_AGG) != 0) + { + bool force_rel_creation; + + /* + * If we're doing partition-wise aggregation at this level, force + * creation of a partially_grouped_rel so we can add partition-wise + * paths to it. + */ + force_rel_creation = (patype == PARTITIONWISE_AGGREGATE_PARTIAL); + partially_grouped_rel = create_partial_grouping_paths(root, grouped_rel, input_rel, gd, - extra); + extra, + force_rel_creation); + } /* * Set partially_grouped_rel_p so that the caller get the newly created @@ -3944,13 +3981,13 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel, *partially_grouped_rel_p = partially_grouped_rel; /* Apply partitionwise aggregation technique, if possible. */ - if ((extra->flags & GROUPING_CAN_PARTITIONWISE_AGG) != 0) + if (patype != PARTITIONWISE_AGGREGATE_NONE) create_partitionwise_grouping_paths(root, input_rel, grouped_rel, partially_grouped_rel, agg_costs, - gd, extra); + gd, patype, extra); /* If we are doing partial aggregation only, return. */ - if (extra->is_partial_aggregation) + if (extra->patype == PARTITIONWISE_AGGREGATE_PARTIAL) { Assert(partially_grouped_rel); @@ -6393,7 +6430,8 @@ create_partial_grouping_paths(PlannerInfo *root, RelOptInfo *grouped_rel, RelOptInfo *input_rel, grouping_sets_data *gd, - GroupPathExtraData *extra) + GroupPathExtraData *extra, + bool force_rel_creation) { Query *parse = root->parse; RelOptInfo *partially_grouped_rel; @@ -6409,11 +6447,13 @@ create_partial_grouping_paths(PlannerInfo *root, /* * Consider whether we should generate partially aggregated non-partial - * paths. We can only do this if we have a non-partial path, and in - * addition the caller must have requested it by setting - * extra->is_partial_aggregation. + * paths. We can only do this if we have a non-partial path, and only if + * the parent of the input rel is performing partial partitionwise + * aggregation. (Note that extra->patype is the type of partitionwise + * aggregation being used at the parent level, not this level.) */ - if (input_rel->pathlist != NIL && extra->is_partial_aggregation) + if (input_rel->pathlist != NIL && + extra->patype == PARTITIONWISE_AGGREGATE_PARTIAL) cheapest_total_path = input_rel->cheapest_total_path; /* @@ -6426,16 +6466,12 @@ create_partial_grouping_paths(PlannerInfo *root, /* * If we can't partially aggregate partial paths, and we can't partially - * aggregate non-partial paths, then there may not be any point to - * creating a new RelOptInfo after all. However, if partitionwise - * aggregate is a possibility and going to perform a partial aggregation, - * then we need to create the RelOptInfo anyway, because the caller may - * want to add Append paths to it. + * aggregate non-partial paths, then don't bother creating the new + * RelOptInfo at all, unless the caller specified force_rel_creation. */ if (cheapest_total_path == NULL && cheapest_partial_path == NULL && - ((extra->flags & GROUPING_CAN_PARTITIONWISE_AGG) == 0 || - !extra->perform_partial_partitionwise_aggregation)) + !force_rel_creation) return NULL; /* @@ -6855,77 +6891,6 @@ apply_scanjoin_target_to_paths(PlannerInfo *root, } } -/* - * can_partitionwise_grouping - * - * Can we use partitionwise grouping technique? - */ -static bool -can_partitionwise_grouping(PlannerInfo *root, - RelOptInfo *input_rel, - RelOptInfo *grouped_rel, - GroupPathExtraData *extra, - grouping_sets_data *gd, - bool *perform_partial_partitionwise_aggregation) -{ - Query *parse = root->parse; - - *perform_partial_partitionwise_aggregation = false; - - /* No, if user disabled partitionwise aggregation. */ - if (!enable_partitionwise_aggregate) - return false; - - /* - * Currently, grouping sets plan does not work with an inheritance subtree - * (see notes in create_groupingsets_plan). Moreover, grouping sets - * implies multiple group by clauses, each of which may not have all - * partition keys. Those sets which have all partition keys will be - * computed completely for each partition, but others will require partial - * aggregation. We will need to apply partitionwise aggregation at each - * derived group by clause and not as a whole-sale strategy. Due to this - * we won't be able to compute "whole" grouping sets here and thus bail - * out. - */ - if (parse->groupingSets || gd) - return false; - - /* - * Nothing to do, if the input relation is not partitioned or it has no - * partitioned relations. - */ - if (!input_rel->part_scheme || !input_rel->part_rels) - return false; - - /* Nothing to do, if the input relation itself is dummy. */ - if (IS_DUMMY_REL(input_rel)) - return false; - - /* - * If partition keys are part of group by clauses, then we can do full - * partitionwise aggregation. Otherwise need to calculate partial - * aggregates for each partition and combine them. - * - * However, if caller forces to perform partial aggregation, then do that - * unconditionally. - */ - *perform_partial_partitionwise_aggregation = (extra->is_partial_aggregation || - !group_by_has_partkey(input_rel, - grouped_rel, - extra->targetList, - parse->groupClause)); - - /* - * If we need to perform partial aggregation for every child but cannot - * compute partial aggregates, no partitionwise grouping is possible. - */ - if (*perform_partial_partitionwise_aggregation && - (extra->flags & GROUPING_CAN_PARTIAL_AGG) == 0) - return false; - - return true; -} - /* * create_partitionwise_grouping_paths * @@ -6951,25 +6916,22 @@ create_partitionwise_grouping_paths(PlannerInfo *root, RelOptInfo *partially_grouped_rel, const AggClauseCosts *agg_costs, grouping_sets_data *gd, + PartitionwiseAggregateType patype, GroupPathExtraData *extra) { - int nparts; + int nparts = input_rel->nparts; int cnt_parts; RelOptInfo **part_rels; List *grouped_live_children = NIL; List *partially_grouped_live_children = NIL; PathTarget *target = extra->target; - nparts = input_rel->nparts; - part_rels = (RelOptInfo **) palloc(nparts * sizeof(RelOptInfo *)); - - /* - * If partial partitionwise aggregation needs to be performed, then we - * must have created a partially_grouped_rel already. - */ - Assert(!extra->perform_partial_partitionwise_aggregation || + Assert(patype != PARTITIONWISE_AGGREGATE_NONE); + Assert(patype != PARTITIONWISE_AGGREGATE_PARTIAL || partially_grouped_rel != NULL); + part_rels = (RelOptInfo **) palloc(nparts * sizeof(RelOptInfo *)); + /* * For full aggregation or grouping, each partition produces a disjoint * groups which can simply be appended and thus we can say that the child @@ -6977,7 +6939,7 @@ create_partitionwise_grouping_paths(PlannerInfo *root, * the partitioning details for this grouped rel. In case of a partial * aggregation, this is not true. */ - if (!extra->perform_partial_partitionwise_aggregation) + if (patype == PARTITIONWISE_AGGREGATE_FULL) { grouped_rel->part_scheme = input_rel->part_scheme; grouped_rel->nparts = nparts; @@ -7026,8 +6988,12 @@ create_partitionwise_grouping_paths(PlannerInfo *root, nappinfos, appinfos); - /* Is partially aggregated result expected from every child? */ - child_extra.is_partial_aggregation = extra->perform_partial_partitionwise_aggregation; + /* + * extra->patype was the value computed for our parent rel; patype + * is the value for this relation. For the child, our value is it's + * parent rel's value. + */ + child_extra.patype = patype; /* * Create grouping relation to hold fully aggregated grouping and/or @@ -7046,17 +7012,6 @@ create_partitionwise_grouping_paths(PlannerInfo *root, continue; } - /* - * Check whether we can perform partitionwise grouping and/or - * aggregation on this child grouped rel. - */ - if (can_partitionwise_grouping(root, child_input_rel, - child_grouped_rel, &child_extra, gd, - &child_extra.perform_partial_partitionwise_aggregation)) - child_extra.flags |= GROUPING_CAN_PARTITIONWISE_AGG; - else - child_extra.flags &= ~GROUPING_CAN_PARTITIONWISE_AGG; - /* * Copy pathtarget from underneath scan/join as we are modifying it * and translate its Vars with respect to this appendrel. We use @@ -7090,7 +7045,7 @@ create_partitionwise_grouping_paths(PlannerInfo *root, child_partially_grouped_rel); } - if (!extra->perform_partial_partitionwise_aggregation) + if (patype == PARTITIONWISE_AGGREGATE_FULL) { part_rels[cnt_parts] = child_grouped_rel; grouped_live_children = lappend(grouped_live_children, @@ -7134,7 +7089,7 @@ create_partitionwise_grouping_paths(PlannerInfo *root, * Now, create append rel for all grouped children and stick them into the * grouped_rel. */ - if (!extra->perform_partial_partitionwise_aggregation) + if (patype == PARTITIONWISE_AGGREGATE_FULL) add_paths_to_append_rel(root, grouped_rel, grouped_live_children); } diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index a920b30def..f2883de1f2 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -2295,6 +2295,24 @@ typedef struct JoinPathExtraData Relids param_source_rels; } JoinPathExtraData; +/* + * What kind of partitionwise aggregation is in use? + * + * PARTITIONWISE_AGGREGATE_NONE: Not used. + * + * PARTITIONWISE_AGGREGATE_FULL: Aggregate each partition separately, and + * append the results. + * + * PARTITIONWISE_AGGREGATE_PARTIAL: Partially aggregate each partition + * separately, append the results, and then finalize aggregation. + */ +typedef enum +{ + PARTITIONWISE_AGGREGATE_NONE, + PARTITIONWISE_AGGREGATE_FULL, + PARTITIONWISE_AGGREGATE_PARTIAL +} PartitionwiseAggregateType; + /* * Struct for extra information passed to subroutines of create_grouping_paths * @@ -2307,9 +2325,7 @@ typedef struct JoinPathExtraData * target_parallel_safe is true if target is parallel safe. * havingQual gives list of quals to be applied post aggregation. * targetList gives list of columns to be projected. - * is_partial_aggregation is true if doing partial aggregation. - * perform_partial_partitionwise_aggregation is true if child needs to perform - * partial partitionwise aggregation. + * patype is the type of partitionwise aggregation that is being performed. */ typedef struct { @@ -2327,9 +2343,7 @@ typedef struct bool target_parallel_safe; Node *havingQual; List *targetList; - - bool is_partial_aggregation; - bool perform_partial_partitionwise_aggregation; + PartitionwiseAggregateType patype; } GroupPathExtraData; /* -- 2.14.3 (Apple Git-98)