Repository: hive Updated Branches: refs/heads/branch-2.0 1420e65f5 -> a73816a13
HIVE-12526 : PerfLogger for hive compiler and optimizer (Hari Subramaniyan, reviewed by Jesus Camacho Rodriguez) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/592179c4 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/592179c4 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/592179c4 Branch: refs/heads/branch-2.0 Commit: 592179c40cc1953ea7ac8a8468c6a5470ffdd4cd Parents: 178a6bf Author: Hari Subramaniyan <harisan...@apache.org> Authored: Mon Dec 14 17:18:34 2015 -0800 Committer: Hari Subramaniyan <harisan...@apache.org> Committed: Fri Dec 18 12:45:07 2015 -0800 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/log/PerfLogger.java | 1 + .../ql/optimizer/BucketMapJoinOptimizer.java | 3 +- .../BucketingSortingReduceSinkOptimizer.java | 2 +- .../hadoop/hive/ql/optimizer/ColumnPruner.java | 2 +- .../hive/ql/optimizer/ConstantPropagate.java | 2 +- .../optimizer/FixedBucketPruningOptimizer.java | 2 +- .../hive/ql/optimizer/GlobalLimitOptimizer.java | 2 +- .../hive/ql/optimizer/GroupByOptimizer.java | 2 +- .../ql/optimizer/IdentityProjectRemover.java | 2 +- .../hadoop/hive/ql/optimizer/JoinReorder.java | 2 +- .../ql/optimizer/LimitPushdownOptimizer.java | 2 +- .../hive/ql/optimizer/MapJoinProcessor.java | 2 +- .../ql/optimizer/NonBlockingOpDeDupProc.java | 2 +- .../hadoop/hive/ql/optimizer/Optimizer.java | 4 +- .../ql/optimizer/PartitionColumnsSeparator.java | 2 +- .../hive/ql/optimizer/PointLookupOptimizer.java | 2 +- .../hadoop/hive/ql/optimizer/SamplePruner.java | 2 +- .../ql/optimizer/SimpleFetchAggregation.java | 2 +- .../hive/ql/optimizer/SimpleFetchOptimizer.java | 2 +- .../hive/ql/optimizer/SkewJoinOptimizer.java | 2 +- .../optimizer/SortedDynPartitionOptimizer.java | 2 +- .../SortedMergeBucketMapJoinOptimizer.java | 2 +- .../hive/ql/optimizer/StatsOptimizer.java | 2 +- .../hadoop/hive/ql/optimizer/Transform.java | 20 ++++++- .../translator/HiveOpConverterPostProc.java | 2 +- .../correlation/CorrelationOptimizer.java | 2 +- .../correlation/ReduceSinkDeDuplication.java | 2 +- .../ql/optimizer/index/RewriteGBUsingIndex.java | 2 +- .../hive/ql/optimizer/lineage/Generator.java | 2 +- .../ListBucketingPruner.java | 2 +- .../annotation/AnnotateWithOpTraits.java | 2 +- .../pcr/PartitionConditionRemover.java | 2 +- .../hive/ql/optimizer/ppr/PartitionPruner.java | 2 +- .../annotation/AnnotateWithStatistics.java | 2 +- .../ql/optimizer/unionproc/UnionProcessor.java | 2 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 57 +++++++++++++++++++- .../hadoop/hive/ql/parse/TezCompiler.java | 6 ++- .../hadoop/hive/ql/ppd/PredicatePushDown.java | 2 +- .../ql/ppd/PredicateTransitivePropagate.java | 2 +- .../hive/ql/ppd/SyntheticJoinPredicate.java | 2 +- 40 files changed, 118 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index 1ef636c..f6eb64b 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -39,6 +39,7 @@ public class PerfLogger { public static final String COMPILE = "compile"; public static final String PARSE = "parse"; public static final String ANALYZE = "semanticAnalyze"; + public static final String OPTIMIZER = "optimizer"; public static final String DO_AUTHORIZATION = "doAuthorization"; public static final String DRIVER_EXECUTE = "Driver.execute"; public static final String INPUT_SUMMARY = "getInputSummary"; http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java index 750427a..a649fdf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; /** * this transformation does bucket map join optimization. */ -public class BucketMapJoinOptimizer implements Transform { +public class BucketMapJoinOptimizer extends Transform { private static final Logger LOG = LoggerFactory.getLogger(GroupByOptimizer.class .getName()); @@ -49,7 +49,6 @@ public class BucketMapJoinOptimizer implements Transform { public BucketMapJoinOptimizer() { } - @Override public ParseContext transform(ParseContext pctx) throws SemanticException { Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java index f59ab4b..312a02c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java @@ -72,7 +72,7 @@ import org.apache.hadoop.hive.shims.ShimLoader; * where T1, T2 and T3 are bucketized/sorted on the same key 'key', we don't need a reducer * to enforce bucketing and sorting */ -public class BucketingSortingReduceSinkOptimizer implements Transform { +public class BucketingSortingReduceSinkOptimizer extends Transform { public BucketingSortingReduceSinkOptimizer() { } http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java index 561b8fc..c353e3e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java @@ -55,7 +55,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; * this changes the row resolver, the tree is built again. This can be optimized * later to patch the tree. */ -public class ColumnPruner implements Transform { +public class ColumnPruner extends Transform { protected ParseContext pGraphContext; /** http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java index 136b5e1..6ff151d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java @@ -59,7 +59,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; * 3. Propagate expression: if the expression is an assignment like column=constant, the expression * will be propagate to parents to see if further folding operation is possible. */ -public class ConstantPropagate implements Transform { +public class ConstantPropagate extends Transform { private static final Logger LOG = LoggerFactory.getLogger(ConstantPropagate.class); protected ParseContext pGraphContext; http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java index c036db0..c63318e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java @@ -60,7 +60,7 @@ import com.google.common.base.Preconditions; * Fixed bucket pruning optimizer goes through all the table scans and annotates them * with a bucketing inclusion bit-set. */ -public class FixedBucketPruningOptimizer implements Transform { +public class FixedBucketPruningOptimizer extends Transform { private static final Log LOG = LogFactory .getLog(FixedBucketPruningOptimizer.class.getName()); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java index 6b04d92..74d6292 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java @@ -61,7 +61,7 @@ import com.google.common.collect.Multimap; * needed by this query. * This optimizer step populates the GlobalLimitCtx which is used later on to prune the inputs. */ -public class GlobalLimitOptimizer implements Transform { +public class GlobalLimitOptimizer extends Transform { private final Logger LOG = LoggerFactory.getLogger(GlobalLimitOptimizer.class.getName()); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java index fe459f7..8f3ce60 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java @@ -69,7 +69,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; * of the bucketing and sorting keys of the underlying table in the same order, the * group by can be be performed on the map-side completely. */ -public class GroupByOptimizer implements Transform { +public class GroupByOptimizer extends Transform { private static final Logger LOG = LoggerFactory.getLogger(GroupByOptimizer.class .getName()); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java index 114c683..48021cb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java @@ -68,7 +68,7 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; * Note absence of select operator after filter and after join operator. * Also, see : identity_proj_remove.q */ -public class IdentityProjectRemover implements Transform { +public class IdentityProjectRemover extends Transform { private static final Logger LOG = LoggerFactory.getLogger(IdentityProjectRemover.class); @Override http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java index e31dc9b..b385c90 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java @@ -37,7 +37,7 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; * largest tag so that they are processed last. In future, once statistics are * implemented, this transformation can also be done based on costs. */ -public class JoinReorder implements Transform { +public class JoinReorder extends Transform { private final Map<Operator<?>, Integer> cache = new IdentityHashMap<Operator<?>, Integer>(); /** http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java index e850550..5a55061 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java @@ -85,7 +85,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; * REDUCER : 10(b,d)-20(l)-30(c,k,o)-40(j,m)-50(n) * LIMIT 3 : 10(b,d)-20(l) */ -public class LimitPushdownOptimizer implements Transform { +public class LimitPushdownOptimizer extends Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java index b4276e4..2e3f930 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java @@ -86,7 +86,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; * the reduce sink operator above the join are converted to map sink operators. In future, once * statistics are implemented, this transformation can also be done based on costs. */ -public class MapJoinProcessor implements Transform { +public class MapJoinProcessor extends Transform { // mapjoin table descriptor contains a key descriptor which needs the field schema // (column type + column name). The column name is not really used anywhere, but it http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java index 2764ae1..37dbe32 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java @@ -54,7 +54,7 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; /** * merges SEL-SEL or FIL-FIL into single operator */ -public class NonBlockingOpDeDupProc implements Transform { +public class NonBlockingOpDeDupProc extends Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 4a7fc0d..8f48e7d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -224,7 +224,9 @@ public class Optimizer { */ public ParseContext optimize() throws SemanticException { for (Transform t : transformations) { - pctx = t.transform(pctx); + t.beginPerfLogging(); + pctx = t.transform(pctx); + t.endPerfLogging(t.toString()); } return pctx; } http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PartitionColumnsSeparator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PartitionColumnsSeparator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PartitionColumnsSeparator.java index f71f37c..aea28e2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PartitionColumnsSeparator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PartitionColumnsSeparator.java @@ -62,7 +62,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; * This predicate is in turn used by the partition pruner to prune the columns that are not * part of the original IN(STRUCT(..)..) predicate. */ -public class PartitionColumnsSeparator implements Transform { +public class PartitionColumnsSeparator extends Transform { private static final Log LOG = LogFactory.getLog(PointLookupOptimizer.class); private static final String IN_UDF = http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java index a1a49cd..44972bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java @@ -64,7 +64,7 @@ import com.google.common.collect.ListMultimap; * AND operator children, the optimization might generate an IN clause that uses * structs. */ -public class PointLookupOptimizer implements Transform { +public class PointLookupOptimizer extends Transform { private static final Logger LOG = LoggerFactory.getLogger(PointLookupOptimizer.class); private static final String IN_UDF = http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java index 2c473b0..dd679db 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java @@ -54,7 +54,7 @@ import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc; * The transformation step that does sample pruning. * */ -public class SamplePruner implements Transform { +public class SamplePruner extends Transform { /** * SamplePrunerCtx. http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java index 39e11a2..f71fd3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java @@ -51,7 +51,7 @@ import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.TableDesc; // execute final aggregation stage for simple fetch query on fetch task -public class SimpleFetchAggregation implements Transform { +public class SimpleFetchAggregation extends Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java index 9b9a5ca..d631007 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java @@ -87,7 +87,7 @@ import org.apache.hadoop.mapred.JobConf; * Tries to convert simple fetch query to single fetch task, which fetches rows directly * from location of table/partition. */ -public class SimpleFetchOptimizer implements Transform { +public class SimpleFetchOptimizer extends Transform { private final Logger LOG = LoggerFactory.getLogger(SimpleFetchOptimizer.class.getName()); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java index 64dc48c..9525271 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java @@ -75,7 +75,7 @@ import org.slf4j.LoggerFactory; * SkewJoinOptimizer. * */ -public class SkewJoinOptimizer implements Transform { +public class SkewJoinOptimizer extends Transform { private static final Logger LOG = LoggerFactory.getLogger(SkewJoinOptimizer.class.getName()); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index c3553a5..d4be5cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -76,7 +76,7 @@ import com.google.common.collect.Maps; * into the destination table. This enables reducers to keep only one record writer all the time * thereby reducing the the memory pressure on the reducers. */ -public class SortedDynPartitionOptimizer implements Transform { +public class SortedDynPartitionOptimizer extends Transform { private static final String BUCKET_NUMBER_COL_NAME = "_bucket_number"; @Override http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java index 5aeeeb8..6aa5335 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java @@ -44,7 +44,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; //try to replace a bucket map join with a sorted merge map join -public class SortedMergeBucketMapJoinOptimizer implements Transform { +public class SortedMergeBucketMapJoinOptimizer extends Transform { private static final Logger LOG = LoggerFactory .getLogger(SortedMergeBucketMapJoinOptimizer.class.getName()); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index ffe706e..03c1c3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -89,7 +89,7 @@ import com.google.common.collect.Lists; * Optimizer looks at query plan to determine if it can answer query using statistics * and than change the plan to answer query entirely using statistics stored in metastore. */ -public class StatsOptimizer implements Transform { +public class StatsOptimizer extends Transform { // TODO: [HIVE-6289] while getting stats from metastore, we currently only get one col at // a time; this could be improved - get all necessary columns in advance, then use local. // TODO: [HIVE-6292] aggregations could be done directly in metastore. Hive over MySQL! http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java index 9f957da..9ce9478 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java @@ -18,8 +18,10 @@ package org.apache.hadoop.hive.ql.optimizer; +import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.session.SessionState; /** * Optimizer interface. All the rule-based optimizations implement this @@ -27,7 +29,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; * current parse context (which contains the operator tree among other things), * perform all the optimizations, and then return the updated parse context. */ -public interface Transform { +public abstract class Transform { /** * All transformation steps implement this interface. * @@ -36,5 +38,19 @@ public interface Transform { * @return ParseContext * @throws SemanticException */ - ParseContext transform(ParseContext pctx) throws SemanticException; + public abstract ParseContext transform(ParseContext pctx) throws SemanticException; + + public void beginPerfLogging() { + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + } + + public void endPerfLogging() { + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER); + } + public void endPerfLogging(String additionalInfo) { + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, additionalInfo); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java index 1d0a254..fc6b99d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java @@ -46,7 +46,7 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -public class HiveOpConverterPostProc implements Transform { +public class HiveOpConverterPostProc extends Transform { private static final Logger LOG = LoggerFactory.getLogger(HiveOpConverterPostProc.class); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java index 1b5d921..e1927e9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java @@ -82,7 +82,7 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; * Test queries associated with this optimizer are correlationoptimizer1.q to * correlationoptimizer14.q */ -public class CorrelationOptimizer implements Transform { +public class CorrelationOptimizer extends Transform { private static final Logger LOG = LoggerFactory.getLogger(CorrelationOptimizer.class.getName()); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java index 583e72f..8da9ad5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java @@ -62,7 +62,7 @@ import com.google.common.collect.Lists; * * This optimizer removes/replaces child-RS (not parent) which is safer way for DefaultGraphWalker. */ -public class ReduceSinkDeDuplication implements Transform { +public class ReduceSinkDeDuplication extends Transform { private static final String RS = ReduceSinkOperator.getOperatorName(); private static final String GBY = GroupByOperator.getOperatorName(); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java index ea1ece6..e59127f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java @@ -89,7 +89,7 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; * For test cases, @see ql_rewrite_gbtoidx.q */ -public class RewriteGBUsingIndex implements Transform { +public class RewriteGBUsingIndex extends Transform { private ParseContext parseContext; private Hive hiveDb; private HiveConf hiveConf; http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java index 82e26d9..0c2ff32 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java @@ -53,7 +53,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; * and tables from the plan before it goes through other * optimization phases. */ -public class Generator implements Transform { +public class Generator extends Transform { /* (non-Javadoc) * @see org.apache.hadoop.hive.ql.optimizer.Transform#transform(org.apache.hadoop.hive.ql.parse.ParseContext) http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java index f399a20..1965120 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; * The transformation step that does list bucketing pruning. * */ -public class ListBucketingPruner implements Transform { +public class ListBucketingPruner extends Transform { static final Logger LOG = LoggerFactory.getLogger(ListBucketingPruner.class.getName()); /* http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java index 0398115..b32d730 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java @@ -49,7 +49,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; * This class annotates each operator with its traits. The OpTraits class * specifies the traits that are populated for each operator. */ -public class AnnotateWithOpTraits implements Transform { +public class AnnotateWithOpTraits extends Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java index 812ca51..7baede9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java @@ -44,7 +44,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; * The transformation step that does partition condition remover. * */ -public class PartitionConditionRemover implements Transform { +public class PartitionConditionRemover extends Transform { // The log private static final Logger LOG = LoggerFactory http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index d99362a..90e9b3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -70,7 +70,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; * The transformation step that does partition pruning. * */ -public class PartitionPruner implements Transform { +public class PartitionPruner extends Transform { // The log public static final String CLASS_NAME = PartitionPruner.class.getName(); http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java index c8b3545..f0c1e86 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; -public class AnnotateWithStatistics implements Transform { +public class AnnotateWithStatistics extends Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java index 547b486..9a53e0c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java @@ -50,7 +50,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; * * The parse Context is not changed. */ -public class UnionProcessor implements Transform { +public class UnionProcessor extends Transform { /** * empty constructor. http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 131414c..9629fb5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -109,6 +109,7 @@ import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; @@ -182,6 +183,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.serde.serdeConstants; @@ -853,7 +855,10 @@ public class CalcitePlanner extends SemanticAnalyzer { this.cluster = cluster; this.relOptSchema = relOptSchema; + PerfLogger perfLogger = SessionState.getPerfLogger(); + // 1. Gen Calcite Plan + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); try { calciteGenPlan = genLogicalPlan(getQB(), true); resultSchema = SemanticAnalyzer.convertRowSchemaToResultSetSchema( @@ -863,6 +868,7 @@ public class CalcitePlanner extends SemanticAnalyzer { semanticException = e; throw new RuntimeException(e); } + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Plan generation"); // Create MD provider HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf); @@ -875,6 +881,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // If join optimizations failed because of missing stats, we continue with // the rest of optimizations if (profilesCBO.contains(ExtendedCBOProfile.JOIN_REORDERING)) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); try { List<RelMetadataProvider> list = Lists.newArrayList(); list.add(mdProvider.getMetadataProvider()); @@ -912,22 +919,26 @@ public class CalcitePlanner extends SemanticAnalyzer { throw e; } } + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Join Reordering"); } else { calciteOptimizedPlan = calcitePreCboPlan; disableSemJoinReordering = false; } // 4. Run other optimizations that do not need stats + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE, UnionMergeRule.INSTANCE, new ProjectMergeRule(false, HiveRelFactories.HIVE_PROJECT_FACTORY), HiveAggregateProjectMergeRule.INSTANCE, HiveJoinCommuteRule.INSTANCE); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Optimizations without stats"); // 5. Run aggregate-join transpose (cost based) // If it failed because of missing stats, we continue with // the rest of optimizations if (conf.getBoolVar(ConfVars.AGGR_JOIN_TRANSPOSE)) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); try { HepProgramBuilder hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP); hepPgmBldr.addRuleInstance(HiveAggregateJoinTransposeRule.INSTANCE); @@ -953,17 +964,21 @@ public class CalcitePlanner extends SemanticAnalyzer { throw e; } } + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Aggregate join transpose"); } // 7. Run rule to fix windowing issue when it is done over // aggregation columns (HIVE-10627) if (profilesCBO.contains(ExtendedCBOProfile.WINDOWING_POSTPROCESSING)) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), HepMatchOrder.BOTTOM_UP, HiveWindowingFixRule.INSTANCE); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Window fixing rule"); } // 8. Run rules to aid in translation from Calcite tree to Hive tree if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); // 8.1. Merge join into multijoin operators (if possible) calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), HepMatchOrder.BOTTOM_UP, HiveJoinProjectTransposeRule.BOTH_PROJECT_INCLUDE_OUTER, @@ -983,6 +998,7 @@ public class CalcitePlanner extends SemanticAnalyzer { calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), HepMatchOrder.BOTTOM_UP, HiveInsertExchange4JoinRule.EXCHANGE_BELOW_JOIN, HiveInsertExchange4JoinRule.EXCHANGE_BELOW_MULTIJOIN); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Translation from Calcite tree to Hive tree"); } if (LOG.isDebugEnabled() && !conf.getBoolVar(ConfVars.HIVE_IN_TEST)) { @@ -1011,14 +1027,19 @@ public class CalcitePlanner extends SemanticAnalyzer { // Partition Pruning; otherwise Expression evaluation may try to execute // corelated sub query. + PerfLogger perfLogger = SessionState.getPerfLogger(); + //1. Distinct aggregate rewrite // Run this optimization early, since it is expanding the operator pipeline. if (!conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("mr") && conf.getBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEDISTINCTREWRITE)) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); // Its not clear, if this rewrite is always performant on MR, since extra map phase // introduced for 2nd MR job may offset gains of this multi-stage aggregation. // We need a cost model for MR to enable this on MR. basePlan = hepPlan(basePlan, true, mdProvider, HiveExpandDistinctAggregatesRule.INSTANCE); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Distinct aggregate rewrite"); } // 2. Try factoring out common filter elements & separating deterministic @@ -1026,14 +1047,18 @@ public class CalcitePlanner extends SemanticAnalyzer { // add on-clauses for old style Join Syntax // Ex: select * from R1 join R2 where ((R1.x=R2.x) and R1.y<10) or // ((R1.x=R2.x) and R1.z=10)) and rand(1) < 0.1 + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, false, mdProvider, HepMatchOrder.ARBITRARY, HivePreFilteringRule.INSTANCE); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, factor out common filter elements and separating deterministic vs non-deterministic UDF"); // 3. PPD for old Join Syntax // NOTE: PPD needs to run before adding not null filters in order to // support old style join syntax (so that on-clauses will get filled up). // TODO: Add in ReduceExpressionrules (Constant folding) to below once // HIVE-11927 is fixed. + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, new HiveFilterProjectTransposeRule( Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY), new HiveFilterSetOpTransposeRule( @@ -1041,6 +1066,8 @@ public class CalcitePlanner extends SemanticAnalyzer { HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule( HiveRelFactories.HIVE_FILTER_FACTORY)); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, PPD for old join syntax"); // TODO: Transitive inference, constant prop & Predicate push down has to // do multiple passes till no more inference is left @@ -1048,14 +1075,19 @@ public class CalcitePlanner extends SemanticAnalyzer { // pred is present below may not be sufficient as inferred & pushed pred // could have been mutated by constant folding/prop // 4. Transitive inference for join on clauses + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, new HiveJoinPushTransitivePredicatesRule( Join.class, HiveRelFactories.HIVE_FILTER_FACTORY)); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Transitive inference for join on clauses"); + // 5. Push down limit through outer join // NOTE: We run this after PPD to support old style join syntax. // Ex: select * from R1 left outer join R2 where ((R1.x=R2.x) and R1.y<10) or // ((R1.x=R2.x) and R1.z=10)) and rand(1) < 0.1 order by R1.x limit 10 if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_JOIN_TRANSPOSE)) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); // This should be a cost based decision, but till we enable the extended cost // model, we will use the given value for the variable final float reductionProportion = HiveConf.getFloatVar(conf, @@ -1067,14 +1099,20 @@ public class CalcitePlanner extends SemanticAnalyzer { basePlan = hepPlan(basePlan, true, mdProvider, HepMatchOrder.BOTTOM_UP, new HiveSortRemoveRule(reductionProportion, reductionTuples), HiveProjectSortTransposeRule.INSTANCE); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Push down limit through outer join"); } // 6. Add not null filters + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, HiveJoinAddNotNullRule.INSTANCE); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Add not null filters"); // 7. Rerun Constant propagation and PPD now that we have added Not NULL filters & did transitive inference // TODO: Add in ReduceExpressionrules (Constant folding) to below once // HIVE-11927 is fixed. + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, new HiveFilterProjectTransposeRule( Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY), new HiveFilterSetOpTransposeRule( @@ -1082,31 +1120,48 @@ public class CalcitePlanner extends SemanticAnalyzer { HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule( HiveRelFactories.HIVE_FILTER_FACTORY)); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Constant propagation and PPD"); // 8. Push Down Semi Joins + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, SemiJoinJoinTransposeRule.INSTANCE, SemiJoinFilterTransposeRule.INSTANCE, SemiJoinProjectTransposeRule.INSTANCE); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Push Down Semi Joins"); // 9. Apply Partition Pruning + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, false, mdProvider, new HivePartitionPruneRule(conf)); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Partition Pruning"); // 10. Projection Pruning (this introduces select above TS & hence needs to be run last due to PP) + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, HiveRelFactories.HIVE_BUILDER.create(cluster, null)); basePlan = fieldTrimmer.trim(basePlan); - + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Projection Pruning"); // 11. Merge Project-Project if possible + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, false, mdProvider, new ProjectMergeRule(true, HiveRelFactories.HIVE_PROJECT_FACTORY)); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Merge Project-Project"); + // 12. Rerun PPD through Project as column pruning would have introduced // DT above scans; By pushing filter just above TS, Hive can push it into // storage (incase there are filters on non partition cols). This only // matches FIL-PROJ-TS + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, new HiveFilterProjectTSTransposeRule( Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, HiveTableScan.class)); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + "Calcite: Prejoin ordering transformation, Rerun PPD"); return basePlan; } http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 147ff7e..ff971ac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption; @@ -86,6 +87,7 @@ import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.TezWork; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; /** @@ -112,7 +114,8 @@ public class TezCompiler extends TaskCompiler { @Override protected void optimizeOperatorPlan(ParseContext pCtx, Set<ReadEntity> inputs, Set<WriteEntity> outputs) throws SemanticException { - + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); // Create the context for the walker OptimizeTezProcContext procCtx = new OptimizeTezProcContext(conf, pCtx, inputs, outputs); @@ -128,6 +131,7 @@ public class TezCompiler extends TaskCompiler { // after the stats phase we might have some cyclic dependencies that we need // to take care of. runCycleAnalysisForPartitionPruning(procCtx, inputs, outputs); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Tez compiler"); } http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java index 2312798..23fa2aa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java @@ -79,7 +79,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; * etc) change the columns as data flows through them. In such cases the column * references are replaced by the corresponding expression in the input data. */ -public class PredicatePushDown implements Transform { +public class PredicatePushDown extends Transform { private static final Logger LOG = LoggerFactory.getLogger(PredicatePushDown.class); private ParseContext pGraphContext; http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java index fb76d5d..b091ebc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java @@ -56,7 +56,7 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; /** * propagates filters to other aliases based on join condition */ -public class PredicateTransitivePropagate implements Transform { +public class PredicateTransitivePropagate extends Transform { private ParseContext pGraphContext; http://git-wip-us.apache.org/repos/asf/hive/blob/592179c4/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java index 7aa91f2..accfb3b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java @@ -60,7 +60,7 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; /** * creates synthetic predicates that represent "IN (keylist other table)" */ -public class SyntheticJoinPredicate implements Transform { +public class SyntheticJoinPredicate extends Transform { private static transient Logger LOG = LoggerFactory.getLogger(SyntheticJoinPredicate.class.getName());
