HIVE-12265: Generate lineage info only if requested (Jimmy, reviewed by Chao)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9c18fce4 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9c18fce4 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9c18fce4 Branch: refs/heads/master Commit: 9c18fce4bfd60288250a90797ce35d042bd91ec7 Parents: efbb041 Author: Jimmy Xiang <jxi...@cloudera.com> Authored: Mon Oct 26 10:20:25 2015 -0700 Committer: Jimmy Xiang <jxi...@cloudera.com> Committed: Wed Oct 28 07:55:21 2015 -0700 ---------------------------------------------------------------------- ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 9 +++++---- .../org/apache/hadoop/hive/ql/optimizer/Optimizer.java | 13 ++++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/9c18fce4/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 3a3fcf1..44b247f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -298,9 +298,7 @@ public class Driver implements CommandProcessor { } public Driver() { - conf = (SessionState.get() != null) ? SessionState.get().getConf() : null; - isParallelEnabled = (conf != null) - && HiveConf.getBoolVar(conf, ConfVars.HIVE_SERVER2_PARALLEL_COMPILATION); + this((SessionState.get() != null) ? SessionState.get().getConf() : null); } /** @@ -1085,6 +1083,10 @@ public class Driver implements CommandProcessor { * while keeping the result around. */ private void releaseResources() { + if (SessionState.get() != null) { + SessionState.get().getLineageState().clear(); + } + if (plan != null) { fetchTask = plan.getFetchTask(); if (fetchTask != null) { @@ -1713,7 +1715,6 @@ public class Driver implements CommandProcessor { if (SessionState.get() != null) { try { - SessionState.get().getLineageState().clear(); SessionState.get().getHiveHistory().logPlanProgress(plan); } catch (Exception e) { // ignore http://git-wip-us.apache.org/repos/asf/hive/blob/9c18fce4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 439f616..25c9618 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.optimizer; import java.util.ArrayList; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -41,6 +42,10 @@ import org.apache.hadoop.hive.ql.ppd.PredicatePushDown; import org.apache.hadoop.hive.ql.ppd.PredicateTransitivePropagate; import org.apache.hadoop.hive.ql.ppd.SyntheticJoinPredicate; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.Sets; + /** * Implementation of the optimizer. */ @@ -67,7 +72,13 @@ public class Optimizer { transformations.add(new HiveOpConverterPostProc()); // Add the transformation that computes the lineage information. - transformations.add(new Generator()); + Set<String> postExecHooks = Sets.newHashSet( + Splitter.on(",").trimResults().omitEmptyStrings().split( + Strings.nullToEmpty(HiveConf.getVar(hiveConf, HiveConf.ConfVars.POSTEXECHOOKS)))); + if (postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.PostExecutePrinter") + || postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.LineageLogger")) { + transformations.add(new Generator()); + } // Try to transform OR predicates in Filter into simpler IN clauses first if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZER)) {