Repository: hive Updated Branches: refs/heads/master 55853b416 -> dfdc6700c
HIVE-11311: Avoid dumping AST tree String in Explain unless necessary (Jesus Camacho Rodriguez, reviewed by Hari Sankar Sivarama Subramaniyan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dfdc6700 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dfdc6700 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dfdc6700 Branch: refs/heads/master Commit: dfdc6700c7a81320a3e0909305645db0248bf01a Parents: 55853b4 Author: Jesus Camacho Rodriguez <jcama...@apache.org> Authored: Tue Jul 21 17:38:39 2015 +0100 Committer: Jesus Camacho Rodriguez <jcama...@apache.org> Committed: Tue Jul 21 17:38:39 2015 +0100 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/ql/Driver.java | 6 ++-- .../apache/hadoop/hive/ql/exec/ExplainTask.java | 7 ++-- .../hive/ql/parse/ExplainSemanticAnalyzer.java | 2 +- .../apache/hadoop/hive/ql/plan/ExplainWork.java | 19 +++++++---- .../parse/TestUpdateDeleteSemanticAnalyzer.java | 34 ++++++++++---------- 5 files changed, 37 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index b74e5fa..6ad3f49 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -482,7 +482,7 @@ public class Driver implements CommandProcessor { } if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) { - String explainOutput = getExplainOutput(sem, plan, tree.dump()); + String explainOutput = getExplainOutput(sem, plan, tree); if (explainOutput != null) { LOG.info("EXPLAIN output for queryid " + queryId + " : " + explainOutput); @@ -533,7 +533,7 @@ public class Driver implements CommandProcessor { * @throws java.io.IOException */ private String getExplainOutput(BaseSemanticAnalyzer sem, QueryPlan plan, - String astStringTree) throws IOException { + ASTNode astTree) throws IOException { String ret = null; ExplainTask task = new ExplainTask(); task.initialize(conf, plan, null); @@ -541,7 +541,7 @@ public class Driver implements CommandProcessor { PrintStream ps = new PrintStream(baos); try { List<Task<?>> rootTasks = sem.getRootTasks(); - task.getJSONPlan(ps, astStringTree, rootTasks, sem.getFetchTask(), false, true, true); + task.getJSONPlan(ps, astTree, rootTasks, sem.getFetchTask(), false, true, true); ret = baos.toString(); } catch (Exception e) { LOG.warn("Exception generating explain output: " + e, e); http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index c6b49bf..21de3cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger; +import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -171,11 +172,11 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { public JSONObject getJSONPlan(PrintStream out, ExplainWork work) throws Exception { - return getJSONPlan(out, work.getAstStringTree(), work.getRootTasks(), work.getFetchTask(), + return getJSONPlan(out, work.getAstTree(), work.getRootTasks(), work.getFetchTask(), work.isFormatted(), work.getExtended(), work.isAppendTaskType()); } - public JSONObject getJSONPlan(PrintStream out, String ast, List<Task<?>> tasks, Task<?> fetchTask, + public JSONObject getJSONPlan(PrintStream out, ASTNode ast, List<Task<?>> tasks, Task<?> fetchTask, boolean jsonOutput, boolean isExtended, boolean appendTaskType) throws Exception { // If the user asked for a formatted output, dump the json output @@ -188,7 +189,7 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable { // Print out the parse AST if (ast != null && isExtended) { - String jsonAST = outputAST(ast, out, jsonOutput, 0); + String jsonAST = outputAST(ast.dump(), out, jsonOutput, 0); if (out != null) { out.println(); } http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index 66d1546..2d365a9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -102,7 +102,7 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer { pCtx, tasks, fetchTask, - input.dump(), + input, sem, extended, formatted, http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java index aa208a5..7a2f883 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java @@ -26,8 +26,9 @@ import java.util.List; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.hooks.ReadEntity; -import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.ParseContext; /** * ExplainWork. @@ -39,6 +40,7 @@ public class ExplainWork implements Serializable { private Path resFile; private ArrayList<Task<? extends Serializable>> rootTasks; private Task<? extends Serializable> fetchTask; + private ASTNode astTree; private String astStringTree; private HashSet<ReadEntity> inputs; private ParseContext pCtx; @@ -63,7 +65,7 @@ public class ExplainWork implements Serializable { ParseContext pCtx, List<Task<? extends Serializable>> rootTasks, Task<? extends Serializable> fetchTask, - String astStringTree, + ASTNode astTree, BaseSemanticAnalyzer analyzer, boolean extended, boolean formatted, @@ -75,7 +77,7 @@ public class ExplainWork implements Serializable { this.resFile = resFile; this.rootTasks = new ArrayList<Task<? extends Serializable>>(rootTasks); this.fetchTask = fetchTask; - this.astStringTree = astStringTree; + this.astTree = astTree; this.analyzer = analyzer; this.inputs = analyzer.getInputs(); this.extended = extended; @@ -112,12 +114,15 @@ public class ExplainWork implements Serializable { this.fetchTask = fetchTask; } - public String getAstStringTree() { - return astStringTree; + public ASTNode getAstTree() { + return astTree; } - public void setAstStringTree(String astStringTree) { - this.astStringTree = astStringTree; + public String getAstStringTree() { + if (astStringTree == null) { + astStringTree = astTree.dump(); + } + return astStringTree; } public HashSet<ReadEntity> getInputs() { http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java index f0435cb..f78f226 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java @@ -59,7 +59,7 @@ public class TestUpdateDeleteSemanticAnalyzer { try { ReturnInfo rc = parseAndAnalyze("insert into table T select a, b from U", "testInsertSelect"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); @@ -70,7 +70,7 @@ public class TestUpdateDeleteSemanticAnalyzer { public void testDeleteAllNonPartitioned() throws Exception { try { ReturnInfo rc = parseAndAnalyze("delete from T", "testDeleteAllNonPartitioned"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -80,7 +80,7 @@ public class TestUpdateDeleteSemanticAnalyzer { public void testDeleteWhereNoPartition() throws Exception { try { ReturnInfo rc = parseAndAnalyze("delete from T where a > 5", "testDeleteWhereNoPartition"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -90,7 +90,7 @@ public class TestUpdateDeleteSemanticAnalyzer { public void testDeleteAllPartitioned() throws Exception { try { ReturnInfo rc = parseAndAnalyze("delete from U", "testDeleteAllPartitioned"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -100,7 +100,7 @@ public class TestUpdateDeleteSemanticAnalyzer { public void testDeleteAllWherePartitioned() throws Exception { try { ReturnInfo rc = parseAndAnalyze("delete from U where a > 5", "testDeleteAllWherePartitioned"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -111,7 +111,7 @@ public class TestUpdateDeleteSemanticAnalyzer { try { ReturnInfo rc = parseAndAnalyze("delete from U where ds = 'today'", "testDeleteFromPartitionOnly"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -122,7 +122,7 @@ public class TestUpdateDeleteSemanticAnalyzer { try { ReturnInfo rc = parseAndAnalyze("delete from U where ds = 'today' and a > 5", "testDeletePartitionWhere"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -132,7 +132,7 @@ public class TestUpdateDeleteSemanticAnalyzer { public void testUpdateAllNonPartitioned() throws Exception { try { ReturnInfo rc = parseAndAnalyze("update T set b = 5", "testUpdateAllNonPartitioned"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -143,7 +143,7 @@ public class TestUpdateDeleteSemanticAnalyzer { try { ReturnInfo rc = parseAndAnalyze("update T set b = 5 where b > 5", "testUpdateAllNonPartitionedWhere"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -153,7 +153,7 @@ public class TestUpdateDeleteSemanticAnalyzer { public void testUpdateAllPartitioned() throws Exception { try { ReturnInfo rc = parseAndAnalyze("update U set b = 5", "testUpdateAllPartitioned"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -164,7 +164,7 @@ public class TestUpdateDeleteSemanticAnalyzer { try { ReturnInfo rc = parseAndAnalyze("update U set b = 5 where b > 5", "testUpdateAllPartitionedWhere"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -175,7 +175,7 @@ public class TestUpdateDeleteSemanticAnalyzer { try { ReturnInfo rc = parseAndAnalyze("update U set b = 5 where ds = 'today'", "testUpdateOnePartition"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -186,7 +186,7 @@ public class TestUpdateDeleteSemanticAnalyzer { try { ReturnInfo rc = parseAndAnalyze("update U set b = 5 where ds = 'today' and b > 5", "testUpdateOnePartitionWhere"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); } @@ -198,7 +198,7 @@ public class TestUpdateDeleteSemanticAnalyzer { ReturnInfo rc = parseAndAnalyze("insert into table T values ('abc', 3), ('ghi', null)", "testInsertValues"); - LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); @@ -212,7 +212,7 @@ public class TestUpdateDeleteSemanticAnalyzer { "('abc', 3, 'today'), ('ghi', 5, 'tomorrow')", "testInsertValuesPartitioned"); - LOG.info(explain((SemanticAnalyzer) rc.sem, rc.plan, rc.ast.dump())); + LOG.info(explain((SemanticAnalyzer) rc.sem, rc.plan, rc.ast)); } finally { cleanupTables(); @@ -285,7 +285,7 @@ public class TestUpdateDeleteSemanticAnalyzer { return new ReturnInfo(tree, sem, plan); } - private String explain(SemanticAnalyzer sem, QueryPlan plan, String astStringTree) throws + private String explain(SemanticAnalyzer sem, QueryPlan plan, ASTNode astTree) throws IOException { FileSystem fs = FileSystem.get(conf); File f = File.createTempFile("TestSemanticAnalyzer", "explain"); @@ -293,7 +293,7 @@ public class TestUpdateDeleteSemanticAnalyzer { fs.create(tmp); fs.deleteOnExit(tmp); ExplainWork work = new ExplainWork(tmp, sem.getParseContext(), sem.getRootTasks(), - sem.getFetchTask(), astStringTree, sem, true, false, false, false, false, false, null); + sem.getFetchTask(), astTree, sem, true, false, false, false, false, false, null); ExplainTask task = new ExplainTask(); task.setWork(work); task.initialize(conf, plan, null);