Repository: incubator-drill Updated Branches: refs/heads/master e5c2da0eb -> caa8b78c5
DRILL-1134 : Fix compilation exception in run-time code, when filter uses two or more array types. Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2ab3dfab Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2ab3dfab Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2ab3dfab Branch: refs/heads/master Commit: 2ab3dfab97a4cd58699919f7c647a7762662759a Parents: e5c2da0 Author: Jinfeng Ni <[email protected]> Authored: Mon Jul 14 16:42:27 2014 -0700 Committer: Jinfeng Ni <[email protected]> Committed: Sun Jul 20 22:20:30 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/expr/EvaluationVisitor.java | 55 +++++++++++++------- .../org/apache/drill/TestExampleQueries.java | 12 ++++- .../complex/writer/TestComplexTypeReader.java | 7 +++ 3 files changed, 54 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ab3dfab/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java index 5511cb2..8bfa149 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java @@ -379,27 +379,30 @@ public class EvaluationVisitor { } else { JExpression vector = e.isSuperReader() ? vv1.component(componentVariable) : vv1; JExpression expr = vector.invoke("getAccessor").invoke("getReader"); - JVar isNull = generator.getEvalBlock().decl(generator.getModel().INT, "isNull", JExpr.lit(0)); + PathSegment seg = e.getReadPath(); + + JVar isNull = null; + boolean isNullReaderLikely = isNullReaderLikely(seg, complex || repeated); + if (isNullReaderLikely) { + isNull = generator.getEvalBlock().decl(generator.getModel().INT, generator.getNextVar("isNull"), JExpr.lit(0)); + } JLabel label = generator.getEvalBlock().label("complex"); JBlock eval = generator.getEvalBlock().block(); // position to the correct value. eval.add(expr.invoke("setPosition").arg(indexVariable)); - PathSegment seg = e.getReadPath(); int listNum = 0; - boolean lastWasArray = false; while (seg != null) { if (seg.isArray()) { - lastWasArray = true; - + // stop once we get to the last segment and the final type is neither complex nor repeated (map, list, repeated list). + // In case of non-complex and non-repeated type, we return Holder, in stead of FieldReader. if (seg.isLastPath() && !complex && !repeated) break; JVar list = generator.declareClassField("list", generator.getModel()._ref(FieldReader.class)); eval.assign(list, expr); - expr = list; // if this is an array, set a single position for the expression to // allow us to read the right data lower down. @@ -413,7 +416,6 @@ public class EvaluationVisitor { currentIndex.lt(desiredIndex) // .cand(list.invoke("next"))).body().assign(currentIndex, currentIndex.plus(JExpr.lit(1))); - expr = list.invoke("reader"); JBlock ifNoVal = eval._if(desiredIndex.ne(currentIndex))._then().block(); if (out.isOptional()) { @@ -422,17 +424,13 @@ public class EvaluationVisitor { ifNoVal.assign(isNull, JExpr.lit(1)); ifNoVal._break(label); + expr = list.invoke("reader"); listNum++; } else { - lastWasArray = false; JExpression fieldName = JExpr.lit(seg.getNameSegment().getPath()); expr = expr.invoke("reader").arg(fieldName); } seg = seg.getChild(); - - // stop once we get to last column or when the segment is an array at - // the end of the reference. - // if(seg == null || seg.isLastPath() && seg.isArray()) break; } if (complex || repeated) { @@ -440,17 +438,20 @@ public class EvaluationVisitor { // // JVar complexReader = generator.declareClassField("reader", generator.getModel()._ref(FieldReader.class)); - JConditional jc = generator.getEvalBlock()._if(isNull.eq(JExpr.lit(0))); + if (isNullReaderLikely) { + JConditional jc = generator.getEvalBlock()._if(isNull.eq(JExpr.lit(0))); - JClass nrClass = generator.getModel().ref(org.apache.drill.exec.vector.complex.impl.NullReader.class); - JExpression nullReader = nrClass.staticRef("INSTANCE"); + JClass nrClass = generator.getModel().ref(org.apache.drill.exec.vector.complex.impl.NullReader.class); + JExpression nullReader = nrClass.staticRef("INSTANCE"); - jc._then().assign(complexReader, expr); - jc._else().assign(complexReader, nullReader); + jc._then().assign(complexReader, expr); + jc._else().assign(complexReader, nullReader); + } else { + eval.assign(complexReader, expr); + } HoldingContainer hc = new HoldingContainer(e.getMajorType(), complexReader, null, null, false, true); return hc; - // //eval.assign(out.getHolder().ref("reader"), expr); } else { if (seg != null) { eval.add(expr.invoke("read").arg(JExpr.lit(seg.getArraySegment().getIndex())).arg(out.getHolder())); @@ -464,6 +465,24 @@ public class EvaluationVisitor { return out; } + /* Check if a Path expression could produce a NullReader. A path expression will produce a null reader, when: + * 1) It contains an array segment as non-leaf segment : a.b[2].c. segment [2] might produce null reader. + * 2) It contains an array segment as leaf segment, AND the final output is complex or repeated : a.b[2], when + * the final type of this expression is a map, or releated list, or repeated map. + */ + private boolean isNullReaderLikely(PathSegment seg, boolean complexOrRepeated) { + while (seg != null) { + if (seg.isArray() && !seg.isLastPath()) + return true; + + if (seg.isLastPath() && complexOrRepeated) + return true; + + seg = seg.getChild(); + } + return false; + } + private HoldingContainer visitReturnValueExpression(ReturnValueExpression e, ClassGenerator<?> generator) { LogicalExpression child = e.getChild(); // Preconditions.checkArgument(child.getMajorType().equals(Types.REQUIRED_BOOLEAN)); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ab3dfab/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java index 21526ed..cfe2a88 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java @@ -103,6 +103,14 @@ public class TestExampleQueries extends BaseTestQuery{ } @Test + public void testFilterOnArrayTypes() throws Exception { + String root = FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString(); + String query = String.format("select columns[0] from dfs_test.`%s` " + + " where cast(columns[0] as int) > 1 and cast(columns[1] as varchar(20))='ASIA'", root); + test(query); + } + + @Test public void testTextPartitions() throws Exception { String root = FileUtils.getResourceAsFile("/store/text/data/").toURI().toString(); String query = String.format("select * from dfs_test.`%s`", root); @@ -116,9 +124,9 @@ public class TestExampleQueries extends BaseTestQuery{ " nations.N_NAME,\n" + " regions.R_NAME\n" + "FROM\n" + - " dfs_test.`[WORKING_PATH]/../../sample-data/nation.parquet` nations\n" + + " cp.`tpch/nation.parquet` nations\n" + "JOIN\n" + - " dfs_test.`[WORKING_PATH]/../../sample-data/region.parquet` regions\n" + + " cp.`tpch/region.parquet` regions\n" + " on nations.N_REGIONKEY = regions.R_REGIONKEY where 1 = 0"); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ab3dfab/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java index 3da3543..e459c4c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java @@ -146,6 +146,13 @@ public class TestComplexTypeReader extends BaseTestQuery{ test(" select convert_to(types[1].minor[0].valueHolder, 'JSON') from cp.`jsoninput/vvtypes.json`;"); } + + @Test + //Two complex type functions in SELECT clause : repeated map (map) --> Json, + public void testA4() throws Exception{ + test(" select convert_to(types[1], 'JSON'), convert_to(modes[2], 'JSON') from cp.`jsoninput/vvtypes.json`;"); + } + @Test //repeated map (map) . public void testB1() throws Exception{
