This is an automated email from the ASF dual-hosted git repository. jcamacho pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
commit 182dac38a472da9fff8be1acb9d250ad5faaa87a Author: Krisztian Kasa <kk...@cloudera.com> AuthorDate: Mon Apr 6 10:49:08 2020 -0700 HIVE-23135: Add RelDistribution trait to HiveSortExchange (Krisztian Kasa, reviewed by Jesus Camacho Rodriguez) --- .../hadoop/hive/ql/optimizer/calcite/TraitsUtil.java | 4 +++- .../ql/optimizer/calcite/cost/HiveVolcanoPlanner.java | 6 +++++- .../calcite/reloperators/HiveSortExchange.java | 10 +++++----- .../rules/HiveProjectSortExchangeTransposeRule.java | 17 +++++++++-------- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java index 8b82241..c41a8c0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java @@ -23,6 +23,7 @@ import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelDistributions; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode; public class TraitsUtil { @@ -32,7 +33,8 @@ public class TraitsUtil { } public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster) { - return cluster.traitSetOf(HiveRelNode.CONVENTION, RelCollations.EMPTY); + return cluster.traitSet() + .replace(HiveRelNode.CONVENTION).replace(RelCollations.EMPTY).replace(RelDistributions.ANY); } public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster, RelTraitSet traitsFromInput) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java index 5dcbff6..208fb15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java @@ -28,10 +28,10 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.plan.volcano.VolcanoPlanner; import org.apache.calcite.rel.RelCollationTraitDef; +import org.apache.calcite.rel.RelDistributionTraitDef; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.util.Util; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfPlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.HivePlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; @@ -46,6 +46,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveDruidRules; */ public class HiveVolcanoPlanner extends VolcanoPlanner { private static final boolean ENABLE_COLLATION_TRAIT = true; + private static final boolean ENABLE_DISTRIBUTION_TRAIT = true; private final boolean isHeuristic; @@ -62,6 +63,9 @@ public class HiveVolcanoPlanner extends VolcanoPlanner { if (ENABLE_COLLATION_TRAIT) { planner.addRelTraitDef(RelCollationTraitDef.INSTANCE); } + if (ENABLE_DISTRIBUTION_TRAIT) { + planner.addRelTraitDef(RelDistributionTraitDef.INSTANCE); + } return planner; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java index 8c46e44..ce794cb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java @@ -61,13 +61,13 @@ public final class HiveSortExchange extends SortExchange implements HiveRelNode RelOptCluster cluster = input.getCluster(); distribution = RelDistributionTraitDef.INSTANCE.canonize(distribution); collation = RelCollationTraitDef.INSTANCE.canonize(collation); - RelTraitSet traitSet = getTraitSet(collation, cluster); + RelTraitSet traitSet = getTraitSet(cluster, collation, distribution); return new HiveSortExchange(cluster, traitSet, input, distribution, collation, keys); } - private static RelTraitSet getTraitSet(RelCollation collation, RelOptCluster cluster) { - // add distribution - return TraitsUtil.getDefaultTraitSet(cluster).replace(collation); + private static RelTraitSet getTraitSet( + RelOptCluster cluster, RelCollation collation, RelDistribution distribution) { + return TraitsUtil.getDefaultTraitSet(cluster).replace(collation).replace(distribution); } public static HiveSortExchange create(RelNode input, @@ -75,7 +75,7 @@ public final class HiveSortExchange extends SortExchange implements HiveRelNode RelOptCluster cluster = input.getCluster(); distribution = RelDistributionTraitDef.INSTANCE.canonize(distribution); collation = RelCollationTraitDef.INSTANCE.canonize(collation); - RelTraitSet traitSet = getTraitSet(collation, cluster); + RelTraitSet traitSet = getTraitSet(cluster, collation, distribution); RelCollation canonizedCollation = traitSet.canonize(RelCollationImpl.of(collation.getFieldCollations())); ImmutableList.Builder<RexNode> builder = ImmutableList.builder(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java index 06cbe3d..d40d5c4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java @@ -29,6 +29,9 @@ import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollationImpl; +import org.apache.calcite.rel.RelCollationTraitDef; +import org.apache.calcite.rel.RelDistribution; +import org.apache.calcite.rel.RelDistributionTraitDef; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.SortExchange; @@ -83,19 +86,17 @@ public final class HiveProjectSortExchangeTransposeRule extends RelOptRule { return; } - RelTraitSet newTraitSet = TraitsUtil.getDefaultTraitSet(sortExchange.getCluster()); - RelCollation newCollation = newTraitSet.canonize(RelCollationImpl.of(fieldCollations)); - newTraitSet = newTraitSet.replace(newCollation); + RelCollation newCollation = RelCollationTraitDef.INSTANCE.canonize(RelCollationImpl.of(fieldCollations)); List<Integer> newDistributionKeys = getNewRelDistributionKeys(project, sortExchange.getDistribution()); + RelDistribution newDistribution = RelDistributionTraitDef.INSTANCE.canonize( + new HiveRelDistribution(sortExchange.getDistribution().getType(), newDistributionKeys)); + RelTraitSet newTraitSet = TraitsUtil.getDefaultTraitSet(sortExchange.getCluster()) + .replace(newCollation).replace(newDistribution); // New operators final RelNode newProject = project.copy(sortExchange.getInput().getTraitSet(), ImmutableList.of(sortExchange.getInput())); - final SortExchange newSort = sortExchange.copy( - newTraitSet, - newProject, - new HiveRelDistribution(sortExchange.getDistribution().getType(), newDistributionKeys), - newCollation); + final SortExchange newSort = sortExchange.copy(newTraitSet, newProject, newDistribution, newCollation); call.transformTo(newSort); }