DRILL-979: For Union-All don't do final column re-ordering since the ordering is already decided by the left input of the Union.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/19dbb023 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/19dbb023 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/19dbb023 Branch: refs/heads/master Commit: 19dbb02361cea87f679a9b65abfecc6d5541aff0 Parents: 1fb5840 Author: Aman Sinha <[email protected]> Authored: Sun Jul 13 17:16:53 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Sun Jul 20 16:27:21 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/planner/physical/AggPrelBase.java | 5 +++++ .../exec/planner/physical/ComplexToJsonPrel.java | 5 +++++ .../drill/exec/planner/physical/FilterPrel.java | 5 +++++ .../apache/drill/exec/planner/physical/JoinPrel.java | 5 +++++ .../drill/exec/planner/physical/LimitPrel.java | 5 +++++ .../org/apache/drill/exec/planner/physical/Prel.java | 1 + .../exec/planner/physical/ProducerConsumerPrel.java | 6 ++++++ .../drill/exec/planner/physical/ProjectPrel.java | 6 +++++- .../apache/drill/exec/planner/physical/ScanPrel.java | 4 ++++ .../drill/exec/planner/physical/ScreenPrel.java | 5 +++++ .../drill/exec/planner/physical/SinglePrel.java | 4 ++++ .../apache/drill/exec/planner/physical/SortPrel.java | 6 ++++++ .../drill/exec/planner/physical/UnionPrel.java | 5 +++++ .../drill/exec/planner/physical/WriterPrel.java | 7 ++++++- .../physical/visitor/FinalColumnReorderer.java | 5 +++++ .../src/test/java/org/apache/drill/TestUnionAll.java | 15 +++++++-------- 16 files changed, 79 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java index 0a3fde8..3a16414 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java @@ -185,5 +185,10 @@ public abstract class AggPrelBase extends AggregateRelBase implements Prel{ public <T, X, E extends Throwable> T accept(PrelVisitor<T, X, E> logicalVisitor, X value) throws E { return logicalVisitor.visitPrel(this, value); } + + @Override + public boolean needsFinalColumnReordering() { + return true; + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ComplexToJsonPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ComplexToJsonPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ComplexToJsonPrel.java index b6bedb6..35e8278 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ComplexToJsonPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ComplexToJsonPrel.java @@ -66,4 +66,9 @@ public class ComplexToJsonPrel extends SingleRel implements Prel { return logicalVisitor.visitPrel(this, value); } + @Override + public boolean needsFinalColumnReordering() { + return true; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java index c857261..337a15f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java @@ -82,4 +82,9 @@ public class FilterPrel extends DrillFilterRelBase implements Prel { return SelectionVectorMode.TWO_BYTE; } + @Override + public boolean needsFinalColumnReordering() { + return true; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java index 0db9a97..cf996dc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java @@ -96,4 +96,9 @@ public abstract class JoinPrel extends DrillJoinRelBase implements Prel{ return proj; } + @Override + public boolean needsFinalColumnReordering() { + return true; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java index 7fad0d9..a4bb503 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java @@ -89,6 +89,11 @@ public class LimitPrel extends DrillLimitRelBase implements Prel { return SelectionVectorMode.TWO_BYTE; } + @Override + public boolean needsFinalColumnReordering() { + return true; + } + // @Override // public LogicalOperator implement(DrillImplementor implementor) { // LogicalOperator inputOp = implementor.visitChild(this, 0, getChild()); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java index a27626a..9286bb3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java @@ -36,5 +36,6 @@ public interface Prel extends DrillRelNode, Iterable<Prel>{ public SelectionVectorMode[] getSupportedEncodings(); public SelectionVectorMode getEncoding(); + boolean needsFinalColumnReordering(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProducerConsumerPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProducerConsumerPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProducerConsumerPrel.java index 35aebdc..6903767 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProducerConsumerPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProducerConsumerPrel.java @@ -75,4 +75,10 @@ public class ProducerConsumerPrel extends SingleRel implements DrillRelNode, Pre public SelectionVectorMode getEncoding() { return SelectionVectorMode.NONE; } + + @Override + public boolean needsFinalColumnReordering() { + return true; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java index 5280469..ae21c2c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java @@ -81,5 +81,9 @@ public class ProjectPrel extends DrillProjectRelBase implements Prel{ return SelectionVectorMode.NONE; } - + @Override + public boolean needsFinalColumnReordering() { + return false; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java index fc62331..e65c110 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java @@ -147,4 +147,8 @@ public class ScanPrel extends AbstractRelNode implements DrillScanPrel { return SelectionVectorMode.NONE; } + @Override + public boolean needsFinalColumnReordering() { + return true; + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java index c713d7c..1a4da8a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java @@ -74,5 +74,10 @@ public class ScreenPrel extends DrillScreenRelBase implements Prel { public SelectionVectorMode getEncoding() { return SelectionVectorMode.NONE; } + + @Override + public boolean needsFinalColumnReordering() { + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SinglePrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SinglePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SinglePrel.java index c23e2a1..3199943 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SinglePrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SinglePrel.java @@ -48,4 +48,8 @@ public abstract class SinglePrel extends SingleRel implements Prel{ return SelectionVectorMode.DEFAULT; } + @Override + public boolean needsFinalColumnReordering() { + return true; + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java index 2a8c7cf..3ef33ae 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java @@ -105,4 +105,10 @@ public class SortPrel extends SortRel implements Prel { public SelectionVectorMode getEncoding() { return SelectionVectorMode.FOUR_BYTE; } + + @Override + public boolean needsFinalColumnReordering() { + return true; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java index cade2df..dd1d018 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java @@ -44,5 +44,10 @@ public abstract class UnionPrel extends DrillUnionRelBase implements Prel{ public Iterator<Prel> iterator() { return PrelUtil.iter(this.getInputs()); } + + @Override + public boolean needsFinalColumnReordering() { + return false; + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java index d4d135c..d9db36e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java @@ -77,5 +77,10 @@ public class WriterPrel extends DrillWriterRelBase implements Prel { public SelectionVectorMode getEncoding() { return SelectionVectorMode.NONE; } - + + @Override + public boolean needsFinalColumnReordering() { + return true; + } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java index 4ea82cd..85f5ba3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java @@ -47,6 +47,11 @@ public class FinalColumnReorderer extends BasePrelVisitor<Prel, Void, RuntimeExc } private Prel addTrivialOrderedProjectPrel(Prel prel){ + + if ( !prel.needsFinalColumnReordering()) { + return prel; + } + RelDataType t = prel.getRowType(); RexBuilder b = prel.getCluster().getRexBuilder(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/19dbb023/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java index 4cb6c3e..c21298e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java @@ -23,11 +23,6 @@ import org.junit.Test; public class TestUnionAll extends BaseTestQuery{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestUnionAll.class); - @Test - public void testQuery1() throws Exception { - test("select c_name from cp.`tpch/customer.parquet` union all select c_acctbal from cp.`tpch/customer.parquet`"); - } - @Test // Simple Union-All over two scans public void testUnionAll1() throws Exception { test("select n_regionkey from cp.`tpch/nation.parquet` union all select r_regionkey from cp.`tpch/region.parquet`"); @@ -53,11 +48,15 @@ public class TestUnionAll extends BaseTestQuery{ test("select * from cp.`tpch/region.parquet` r1 union all select * from cp.`tpch/region.parquet` r2"); } - @Test - @Ignore // Produces wrong result + @Test // Union-All where same column is projected twice in right child public void testUnionAll6() throws Exception { test("select n_nationkey, n_regionkey from cp.`tpch/nation.parquet` where n_regionkey = 1 union all select r_regionkey, r_regionkey from cp.`tpch/region.parquet` where r_regionkey = 2"); - } + } + + @Test // Union-All where same column is projected twice in left and right child + public void testUnionAll6_1() throws Exception { + test("select n_nationkey, n_nationkey from cp.`tpch/nation.parquet` union all select r_regionkey, r_regionkey from cp.`tpch/region.parquet`"); + } @Test // Union-all of two string literals of different lengths public void testUnionAll7() throws Exception {
