Fix EXPLAIN plan which was broken in recent change, add test case for EXPLAIN. Move debug msg in DrillSqlWorker to logger.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/4f98a4f4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/4f98a4f4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/4f98a4f4 Branch: refs/heads/master Commit: 4f98a4f41802771a487671382c3b189cd9660b1b Parents: 56411a5 Author: Jinfeng Ni <[email protected]> Authored: Wed Apr 9 16:18:50 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Sat Apr 19 18:07:12 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/planner/sql/DrillSqlWorker.java | 115 +++++++++++++------ .../apache/drill/exec/work/foreman/Foreman.java | 18 ++- .../org/apache/drill/TestExampleQueries.java | 10 ++ 3 files changed, 108 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4f98a4f4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index e796f12..d5ad1fc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -60,6 +60,7 @@ import org.eigenbase.sql.SqlNode; import org.eigenbase.sql.parser.SqlParseException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.hive12.common.base.Preconditions; public class DrillSqlWorker { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSqlWorker.class); @@ -81,7 +82,7 @@ public class DrillSqlWorker { // this.planner = Frameworks.getPlanner(Lex.MYSQL, SqlParserImpl.FACTORY, schemaFactory, SqlStdOperatorTable.instance(), traitDefs, RULES); } - private class RelResult{ + public class RelResult{ final ResultMode mode; final RelNode node; public RelResult(ResultMode mode, RelNode node) { @@ -89,18 +90,28 @@ public class DrillSqlWorker { this.mode = mode; this.node = node; } + + public ResultMode getMode() { + return this.mode; + } } /* - * Return the logical DrillRel tree + * Given a SQL string, return the logical DrillRel tree, plus mode (execute, or EXPLAIN mode). */ - private RelResult getRel(String sql) throws SqlParseException, ValidationException, RelConversionException{ - SqlNode sqlNode = planner.parse(sql); - + public RelResult getLogicalRel(String sql) throws SqlParseException, ValidationException, RelConversionException{ + if(logger.isDebugEnabled()) { + logger.debug("SQL : " + sql); + } + + // Call optiq to parse the SQL string. + SqlNode sqlNode = planner.parse(sql); ResultMode resultMode = ResultMode.EXEC; + //Process EXPLAIN if(sqlNode.getKind() == SqlKind.EXPLAIN){ SqlExplain explain = (SqlExplain) sqlNode; + sqlNode = explain.operand(0); SqlExplain.Depth depth = (SqlExplain.Depth) explain.getDepth(); switch(depth){ case LOGICAL: @@ -113,11 +124,16 @@ public class DrillSqlWorker { } } + // Call optiq to validate SqlNode tree and convert it to RelNode tree. SqlNode validatedNode = planner.validate(sqlNode); RelNode relNode = planner.convert(validatedNode); - System.out.println(RelOptUtil.toString(relNode, SqlExplainLevel.ALL_ATTRIBUTES)); + //Debug + if(logger.isDebugEnabled()) { + logger.debug("RelNode tree : " + RelOptUtil.toString(relNode, SqlExplainLevel.ALL_ATTRIBUTES)); + } + // Call optiq to transform RelNode into Drill Logical RelNode tree. RelNode convertedRelNode = planner.transform(LOGICAL_RULES, relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL), relNode); if(convertedRelNode instanceof DrillStoreRel){ throw new UnsupportedOperationException(); @@ -125,15 +141,35 @@ public class DrillSqlWorker { convertedRelNode = new DrillScreenRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode); } - System.out.println(RelOptUtil.toString(convertedRelNode, SqlExplainLevel.ALL_ATTRIBUTES)); + //Debug + if(logger.isDebugEnabled()) { + logger.debug("Drill LogicalRel tree : " + RelOptUtil.toString(convertedRelNode, SqlExplainLevel.ALL_ATTRIBUTES)); + } return new RelResult(resultMode, convertedRelNode); } - - + /* + * Given a Drill LogicalRel tree, return Drill Logical Plan. + * @param relResult : RelResult whose node is the root of Drill logicalrel tree. + */ + public LogicalPlan getLogicalPlan(RelResult relResult) throws SqlParseException, ValidationException, RelConversionException{ + RelNode logicalRelRoot = relResult.node; + + Preconditions.checkArgument(logicalRelRoot.getConvention() == DrillRel.DRILL_LOGICAL); + + DrillImplementor implementor = new DrillImplementor(new DrillParseContext(), relResult.mode); + implementor.go( (DrillRel) logicalRelRoot); + planner.close(); + planner.reset(); + return implementor.getPlan(); + } + + /* + * Given a SQL string, return the Drill logical plan. + */ public LogicalPlan getLogicalPlan(String sql) throws SqlParseException, ValidationException, RelConversionException{ - RelResult result = getRel(sql); + RelResult result = getLogicalRel(sql); RelNode convertedRelNode = planner.transform(LOGICAL_RULES, result.node.getTraitSet().plus(DrillRel.DRILL_LOGICAL), result.node); if(convertedRelNode instanceof DrillStoreRel){ @@ -145,22 +181,51 @@ public class DrillSqlWorker { implementor.go( (DrillRel) convertedRelNode); planner.close(); planner.reset(); - return implementor.getPlan(); + return implementor.getPlan(); + } + + /* + * Given a Drill LogicalRel tree, return Drill Physical plan. + * @param relResult : RelResult whose node is the root of Drill logicalrel tree. + * @param qcontext : QueryContext used by PhysicalPlanCreator. + */ + public PhysicalPlan getPhysicalPlan(RelResult relResult, QueryContext qcontext) throws SqlParseException, ValidationException, RelConversionException, IOException { + RelNode logicalRelRoot = relResult.node; + + Preconditions.checkArgument(logicalRelRoot.getConvention() == DrillRel.DRILL_LOGICAL); + + RelTraitSet traits = logicalRelRoot.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON); + Prel phyRelNode = (Prel) planner.transform(PHYSICAL_MEM_RULES, traits, logicalRelRoot); + + //Debug + if(logger.isDebugEnabled()) { + String msg = RelOptUtil.toString(phyRelNode, SqlExplainLevel.ALL_ATTRIBUTES); + logger.debug("Drill PhysicalRel tree: " + msg); + } + PhysicalPlanCreator pplanCreator = new PhysicalPlanCreator(qcontext); + PhysicalPlan plan = pplanCreator.build(phyRelNode, true /* rebuild */); + + planner.close(); + planner.reset(); + return plan; } + /* + * Given a SQL string, return Drill physical plan. + */ public PhysicalPlan getPhysicalPlan(String sql, QueryContext qcontext) throws SqlParseException, ValidationException, RelConversionException, IOException { - RelResult result = getRel(sql); + RelResult result = getLogicalRel(sql); RelTraitSet traits = result.node.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON); Prel phyRelNode = (Prel) planner.transform(PHYSICAL_MEM_RULES, traits, result.node); //Debug. - System.err.println("SQL : " + sql); - logger.debug("SQL : " + sql); - String msg = RelOptUtil.toString(phyRelNode, SqlExplainLevel.ALL_ATTRIBUTES); - System.out.println(msg); - logger.debug(msg); + if(logger.isDebugEnabled()) { + logger.debug("SQL : " + sql); + String msg = RelOptUtil.toString(phyRelNode, SqlExplainLevel.ALL_ATTRIBUTES); + logger.debug("Drill PhysicalRel tree: " + msg); + } PhysicalPlanCreator pplanCreator = new PhysicalPlanCreator(qcontext); PhysicalPlan plan = pplanCreator.build(phyRelNode, true /* rebuild */); @@ -171,20 +236,4 @@ public class DrillSqlWorker { } - public void runPhysicalPlan(PhysicalPlan phyPlan, DrillConfig config) { - QuerySubmitter qs = new QuerySubmitter(); - - ObjectMapper mapper = config.getMapper(); - - try { - String phyPlanStr = mapper.writeValueAsString(phyPlan); - - System.out.println(phyPlanStr); - - qs.submitQuery(null, phyPlanStr, "physical", null, true, 1, "csv"); - } catch (Exception e) { - System.err.println("Query fails " + e.toString()); - } - } - } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4f98a4f4/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index 858508b..4dfb309 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -47,6 +47,7 @@ import org.apache.drill.exec.planner.fragment.PlanningSet; import org.apache.drill.exec.planner.fragment.SimpleParallelizer; import org.apache.drill.exec.planner.fragment.StatsCollector; import org.apache.drill.exec.planner.sql.DrillSqlWorker; +import org.apache.drill.exec.planner.sql.DrillSqlWorker.RelResult; import org.apache.drill.exec.proto.BitControl.PlanFragment; import org.apache.drill.exec.proto.GeneralRPCProtos.Ack; import org.apache.drill.exec.proto.UserBitShared.DrillPBError; @@ -348,11 +349,24 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{ try{ DrillSqlWorker sqlWorker = new DrillSqlWorker(context.getFactory(), context.getFunctionRegistry()); - PhysicalPlan physical = sqlWorker.getPhysicalPlan(sql, context); + RelResult relResult = sqlWorker.getLogicalRel(sql); + //EXPLAIN logical + if (relResult.getMode() == ResultMode.LOGICAL) { + returnLogical(sqlWorker.getLogicalPlan(relResult)); + return; + } + + PhysicalPlan physical = sqlWorker.getPhysicalPlan(relResult, context); + if(logger.isDebugEnabled()) { logger.debug("Distributed Physical {}", context.getConfig().getMapper().writeValueAsString(physical)); - System.out.println(context.getConfig().getMapper().writeValueAsString(physical)); + } + + //EXPLAIN physical + if (relResult.getMode() == ResultMode.PHYSICAL) { + returnPhysical(physical); + return; } runPhysicalPlan(physical); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4f98a4f4/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java index 5baaf63..35c4707 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java @@ -56,6 +56,16 @@ public class TestExampleQueries { public void testGroupBy() throws Exception{ test("select marital_status, COUNT(1) as cnt from cp.`employee.json` group by marital_status"); } + + @Test + public void testExpalinPhysical() throws Exception{ + test("explain plan for select marital_status, COUNT(1) as cnt from cp.`employee.json` group by marital_status"); + } + + @Test + public void testExpalinLogical() throws Exception{ + test("explain plan without implementation for select marital_status, COUNT(1) as cnt from cp.`employee.json` group by marital_status"); + } private void test(String sql) throws Exception{ boolean good = false;
