HIVE-11569 : Use PreOrderOnceWalker where feasible (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e1b88a19 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e1b88a19 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e1b88a19 Branch: refs/heads/llap Commit: e1b88a19a98534cb7c2f6d46e39f5eb135ab6b50 Parents: 6433c3b Author: Ashutosh Chauhan <hashut...@apache.org> Authored: Tue Aug 18 09:13:59 2015 -0700 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Tue Aug 18 09:13:59 2015 -0700 ---------------------------------------------------------------------- .../hive/ql/optimizer/index/RewriteCanApplyCtx.java | 8 ++++---- .../optimizer/physical/NullScanTaskDispatcher.java | 6 +++--- .../hive/ql/optimizer/physical/Vectorizer.java | 16 +++------------- .../hadoop/hive/ql/ppd/SyntheticJoinPredicate.java | 4 ++-- 4 files changed, 12 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java index b56b608..988bb29 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.PreOrderWalker; +import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -75,7 +75,7 @@ public final class RewriteCanApplyCtx implements NodeProcessorCtx { private String baseTableName; private String indexTableName; private String aggFunction; - + private TableScanOperator tableScanOperator; private List<SelectOperator> selectOperators; private List<GroupByOperator> groupByOperators; @@ -156,7 +156,7 @@ public final class RewriteCanApplyCtx implements NodeProcessorCtx { /** * This method walks all the nodes starting from topOp TableScanOperator node * and invokes methods from {@link RewriteCanApplyProcFactory} for each of the rules - * added to the opRules map. We use the {@link PreOrderWalker} for a pre-order + * added to the opRules map. We use the {@link PreOrderOnceWalker} for a pre-order * traversal of the operator tree. * * The methods from {@link RewriteCanApplyProcFactory} set appropriate values in @@ -180,7 +180,7 @@ public final class RewriteCanApplyCtx implements NodeProcessorCtx { // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, this); - GraphWalker ogw = new PreOrderWalker(disp); + GraphWalker ogw = new PreOrderOnceWalker(disp); // Create a list of topop nodes List<Node> topNodes = new ArrayList<Node>(); http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java index f901812..8dcd283 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.lib.Dispatcher; import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; -import org.apache.hadoop.hive.ql.lib.PreOrderWalker; +import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer.WalkerCtx; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -61,7 +61,7 @@ import org.apache.hadoop.hive.serde2.NullStructSerDe; public class NullScanTaskDispatcher implements Dispatcher { static final Log LOG = LogFactory.getLog(NullScanTaskDispatcher.class.getName()); - + private final PhysicalContext physicalContext; private final Map<Rule, NodeProcessor> rules; @@ -177,7 +177,7 @@ public class NullScanTaskDispatcher implements Dispatcher { // The dispatcher fires the processor corresponding to the closest // matching rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(null, rules, walkerCtx); - GraphWalker ogw = new PreOrderWalker(disp); + GraphWalker ogw = new PreOrderOnceWalker(disp); // Create a list of topOp nodes ArrayList<Node> topNodes = new ArrayList<Node>(); http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 7ecd50a..2e3bd76 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey; import org.apache.hadoop.hive.ql.exec.spark.SparkTask; import org.apache.hadoop.hive.ql.exec.tez.TezTask; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyLongOperator; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyMultiKeyOperator; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyStringOperator; @@ -68,6 +67,7 @@ import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; @@ -155,7 +155,6 @@ public class Vectorizer implements PhysicalPlanResolver { Set<String> supportedAggregationUdfs = new HashSet<String>(); - private PhysicalContext physicalContext = null; private HiveConf hiveConf; public Vectorizer() { @@ -307,13 +306,10 @@ public class Vectorizer implements PhysicalPlanResolver { class VectorizationDispatcher implements Dispatcher { - private final PhysicalContext physicalContext; - private List<String> reduceColumnNames; private List<TypeInfo> reduceTypeInfos; public VectorizationDispatcher(PhysicalContext physicalContext) { - this.physicalContext = physicalContext; reduceColumnNames = null; reduceTypeInfos = null; } @@ -428,7 +424,7 @@ public class Vectorizer implements PhysicalPlanResolver { MapWorkVectorizationNodeProcessor vnp = new MapWorkVectorizationNodeProcessor(mapWork, isTez); addMapWorkRules(opRules, vnp); Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null); - GraphWalker ogw = new PreOrderWalker(disp); + GraphWalker ogw = new PreOrderOnceWalker(disp); // iterator the mapper operator tree ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(mapWork.getAliasToWork().values()); @@ -701,12 +697,10 @@ public class Vectorizer implements PhysicalPlanResolver { class MapWorkVectorizationNodeProcessor extends VectorizationNodeProcessor { - private final MapWork mWork; private final boolean isTez; public MapWorkVectorizationNodeProcessor(MapWork mWork, boolean isTez) { super(); - this.mWork = mWork; this.isTez = isTez; } @@ -765,7 +759,7 @@ public class Vectorizer implements PhysicalPlanResolver { private final List<String> reduceColumnNames; private final List<TypeInfo> reduceTypeInfos; - private boolean isTez; + private final boolean isTez; private Operator<? extends OperatorDesc> rootVectorOp; @@ -865,7 +859,6 @@ public class Vectorizer implements PhysicalPlanResolver { @Override public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException { - this.physicalContext = physicalContext; hiveConf = physicalContext.getConf(); boolean vectorPath = HiveConf.getBoolVar(hiveConf, @@ -1406,7 +1399,6 @@ public class Vectorizer implements PhysicalPlanResolver { int[] smallTableIndices; int smallTableIndicesSize; - List<ExprNodeDesc> smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable); if (desc.getValueIndices() != null && desc.getValueIndices().get(posSingleVectorMapJoinSmallTable) != null) { smallTableIndices = desc.getValueIndices().get(posSingleVectorMapJoinSmallTable); LOG.info("Vectorizer isBigTableOnlyResults smallTableIndices " + Arrays.toString(smallTableIndices)); @@ -1445,8 +1437,6 @@ public class Vectorizer implements PhysicalPlanResolver { Operator<? extends OperatorDesc> vectorOp = null; Class<? extends Operator<?>> opClass = null; - boolean isOuterJoin = !desc.getNoOuterJoin(); - VectorMapJoinDesc.HashTableImplementationType hashTableImplementationType = HashTableImplementationType.NONE; VectorMapJoinDesc.HashTableKind hashTableKind = HashTableKind.NONE; VectorMapJoinDesc.HashTableKeyType hashTableKeyType = HashTableKeyType.NONE; http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java index 32af813..44159c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.PreOrderWalker; +import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.optimizer.Transform; @@ -92,7 +92,7 @@ public class SyntheticJoinPredicate implements Transform { // rule and passes the context along SyntheticContext context = new SyntheticContext(pctx); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context); - GraphWalker ogw = new PreOrderWalker(disp); + GraphWalker ogw = new PreOrderOnceWalker(disp); // Create a list of top op nodes List<Node> topNodes = new ArrayList<Node>();
