Repository: incubator-drill Updated Branches: refs/heads/master a147bfce0 -> 90c12c8ce
DRILL-1673: Set the correct group count in splitAndTransferTo for repeated vectors Handle the case in FlattedRecordBatch where we return null internal vectors for repeated lists in the first schema batch Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/90c12c8c Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/90c12c8c Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/90c12c8c Branch: refs/heads/master Commit: 90c12c8ce93fb1451d913f4af4560bd0fb6a5306 Parents: a147bfc Author: Mehant Baid <[email protected]> Authored: Tue Nov 18 01:06:58 2014 -0800 Committer: Mehant Baid <[email protected]> Committed: Wed Nov 19 10:56:26 2014 -0800 ---------------------------------------------------------------------- .../codegen/templates/RepeatedValueVectors.java | 2 +- .../impl/flatten/FlattenRecordBatch.java | 22 ++++++++++++++------ .../complex/writer/TestComplexTypeReader.java | 5 +++++ 3 files changed, 22 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/90c12c8c/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java index bf93d00..3d50b00 100644 --- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java @@ -124,7 +124,7 @@ public final class Repeated${minor.class}Vector extends BaseValueVector implemen } to.parentValueCount = groups; to.childValueCount = valuesToCopy; - m.setValueCount(groups); + m.setValueCount(groups == 0 ? 0 : groups + 1); } private class TransferImpl implements TransferPair{ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/90c12c8c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java index 66c6168..78c1c50 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java @@ -29,6 +29,7 @@ import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.logical.data.NamedExpression; +import org.apache.drill.common.types.TypeProtos; import org.apache.drill.exec.exception.ClassTransformationException; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.ClassGenerator; @@ -263,7 +264,11 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> { ValueVector vector = container.addOrGet(vw.getField()); container.add(vector); } else { - container.add(getFlattenFieldTransferPair().getTo()); + TransferPair pair = getFlattenFieldTransferPair(); + if (pair == null) { + continue; + } + container.add(pair.getTo()); } } else { ValueVector vector = container.addOrGet(vw.getField()); @@ -297,12 +302,15 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> { popConfig.getColumn()).getFieldIds()[0]).getValueClass(), incoming.getValueVectorId(popConfig.getColumn()).getFieldIds()).getValueVector(); - TransferPair tp; + TransferPair tp = null; if (flattenField instanceof RepeatedMapVector) { tp = ((RepeatedMapVector)flattenField).getTransferPairToSingleMap(); } else { ValueVector vvIn = ((RepeatedVector)flattenField).getAccessor().getAllChildValues(); - tp = vvIn.getTransferPair(); + // vvIn may be null because of fast schema return for repeated list vectors + if (vvIn != null) { + tp = vvIn.getTransferPair(); + } } return tp; } @@ -326,9 +334,11 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> { TransferPair tp = getFlattenFieldTransferPair(); - transfers.add(tp); - container.add(tp.getTo()); - transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]); + if (tp != null) { + transfers.add(tp); + container.add(tp.getTo()); + transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]); + } logger.debug("Added transfer for project expression."); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/90c12c8c/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java index df92dde..83e1130 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java @@ -188,4 +188,9 @@ public class TestComplexTypeReader extends BaseTestQuery{ test("select kvgen(x) from cp.`jsoninput/input2.json`"); test("select kvgen(bigintegercol), kvgen(float8col) from cp.`jsoninput/input3.json`"); } + + @Test + public void testNestedFlatten() throws Exception { + test("select flatten(rl) from cp.`jsoninput/input2.json`"); + } }
