HIVE-11252 : CBO (Calcite Return Path): DUMMY project in plan (Jesus Camacho Rodriguez via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/999e0e36 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/999e0e36 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/999e0e36 Branch: refs/heads/beeline-cli Commit: 999e0e3616525d77cf46c5865f9981b5a6b5609a Parents: 90a2cf9 Author: Jesus Camacho Rodriguez <jcama...@apache.org> Authored: Tue Jul 14 08:22:00 2015 +0700 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Tue Jul 14 11:18:33 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/CalcitePlanner.java | 64 +++++++++----------- 1 file changed, 30 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/999e0e36/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 84bb951..1ea236b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -863,38 +863,20 @@ public class CalcitePlanner extends SemanticAnalyzer { calciteOptimizedPlan = hepPlanner.findBestExp(); // 4. Run rule to try to remove projects on top of join operators - hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP); - hepPgmBldr.addRuleInstance(HiveJoinCommuteRule.INSTANCE); - hepPlanner = new HepPlanner(hepPgmBldr.build()); - hepPlanner.registerMetadataProviders(list); - cluster.setMetadataProvider(new CachingRelMetadataProvider(chainedProvider, hepPlanner)); - hepPlanner.setRoot(calciteOptimizedPlan); - calciteOptimizedPlan = hepPlanner.findBestExp(); + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), + HepMatchOrder.BOTTOM_UP, HiveJoinCommuteRule.INSTANCE); // 5. Run rule to fix windowing issue when it is done over // aggregation columns (HIVE-10627) - hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP); - hepPgmBldr.addRuleInstance(HiveWindowingFixRule.INSTANCE); - hepPlanner = new HepPlanner(hepPgmBldr.build()); - hepPlanner.registerMetadataProviders(list); - cluster.setMetadataProvider(new CachingRelMetadataProvider(chainedProvider, hepPlanner)); - hepPlanner.setRoot(calciteOptimizedPlan); - calciteOptimizedPlan = hepPlanner.findBestExp(); + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), + HepMatchOrder.BOTTOM_UP, HiveWindowingFixRule.INSTANCE); // 6. Run rules to aid in translation from Calcite tree to Hive tree if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { // 6.1. Merge join into multijoin operators (if possible) - hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP); - hepPgmBldr.addRuleInstance(HiveJoinToMultiJoinRule.INSTANCE); - hepPgmBldr = hepPgmBldr.addRuleCollection(ImmutableList.of( - HiveJoinProjectTransposeRule.BOTH_PROJECT, - HiveJoinToMultiJoinRule.INSTANCE, - HiveProjectMergeRule.INSTANCE)); - hepPlanner = new HepPlanner(hepPgmBldr.build()); - hepPlanner.registerMetadataProviders(list); - cluster.setMetadataProvider(new CachingRelMetadataProvider(chainedProvider, hepPlanner)); - hepPlanner.setRoot(calciteOptimizedPlan); - calciteOptimizedPlan = hepPlanner.findBestExp(); + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), + HepMatchOrder.BOTTOM_UP, HiveJoinProjectTransposeRule.BOTH_PROJECT, + HiveJoinToMultiJoinRule.INSTANCE, HiveProjectMergeRule.INSTANCE); // The previous rules can pull up projections through join operators, // thus we run the field trimmer again to push them back down HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, HiveProject.DEFAULT_PROJECT_FACTORY, @@ -902,16 +884,14 @@ public class CalcitePlanner extends SemanticAnalyzer { HiveSemiJoin.HIVE_SEMIJOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY, HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY); calciteOptimizedPlan = fieldTrimmer.trim(calciteOptimizedPlan); + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), + HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE, + new ProjectMergeRule(false, HiveProject.DEFAULT_PROJECT_FACTORY)); // 6.2. Introduce exchange operators below join/multijoin operators - hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP); - hepPgmBldr.addRuleInstance(HiveInsertExchange4JoinRule.EXCHANGE_BELOW_JOIN); - hepPgmBldr.addRuleInstance(HiveInsertExchange4JoinRule.EXCHANGE_BELOW_MULTIJOIN); - hepPlanner = new HepPlanner(hepPgmBldr.build()); - hepPlanner.registerMetadataProviders(list); - cluster.setMetadataProvider(new CachingRelMetadataProvider(chainedProvider, hepPlanner)); - hepPlanner.setRoot(calciteOptimizedPlan); - calciteOptimizedPlan = hepPlanner.findBestExp(); + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), + HepMatchOrder.BOTTOM_UP, HiveInsertExchange4JoinRule.EXCHANGE_BELOW_JOIN, + HiveInsertExchange4JoinRule.EXCHANGE_BELOW_MULTIJOIN); } if (LOG.isDebugEnabled() && !conf.getBoolVar(ConfVars.HIVE_IN_TEST)) { @@ -1006,11 +986,27 @@ public class CalcitePlanner extends SemanticAnalyzer { */ private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, RelMetadataProvider mdProvider, RelOptRule... rules) { + return hepPlan(basePlan, followPlanChanges, mdProvider, + HepMatchOrder.TOP_DOWN, rules); + } + + /** + * Run the HEP Planner with the given rule set. + * + * @param basePlan + * @param followPlanChanges + * @param mdProvider + * @param order + * @param rules + * @return optimized RelNode + */ + private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, RelMetadataProvider mdProvider, + HepMatchOrder order, RelOptRule... rules) { RelNode optimizedRelNode = basePlan; HepProgramBuilder programBuilder = new HepProgramBuilder(); if (followPlanChanges) { - programBuilder.addMatchOrder(HepMatchOrder.TOP_DOWN); + programBuilder.addMatchOrder(order); programBuilder = programBuilder.addRuleCollection(ImmutableList.copyOf(rules)); } else { // TODO: Should this be also TOP_DOWN?