Zero out RepeatedMap and RepeatedList offset vectors. Advance var* lastSet when using copy safe.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/85a2e042 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/85a2e042 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/85a2e042 Branch: refs/heads/master Commit: 85a2e0427a91f39d94a77d6aa4af25f895ec93d7 Parents: d468f6d Author: Jacques Nadeau <[email protected]> Authored: Thu May 15 09:18:39 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Thu May 15 09:54:54 2014 -0700 ---------------------------------------------------------------------- .../codegen/templates/NullableValueVectors.java | 55 +++++++++----------- .../exec/vector/complex/RepeatedListVector.java | 1 + .../exec/vector/complex/RepeatedMapVector.java | 4 ++ .../exec/store/json/JsonRecordReader2Test.java | 8 +-- pom.xml | 4 +- 5 files changed, 37 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/85a2e042/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java index 6086c60..61c488a 100644 --- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java @@ -284,30 +284,25 @@ public final class ${className} extends BaseValueVector implements <#if type.maj protected void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){ if (!from.getAccessor().isNull(fromIndex)) { - mutator.set(thisIndex, from.getAccessor().get(fromIndex)); + mutator.set(thisIndex, from.getAccessor().get(fromIndex)); } } + public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ - boolean success = values.copyFromSafe(fromIndex, thisIndex, from); - bits.getMutator().set(thisIndex, 1); - <#if type.major == "VarLen"> - if (success) { - mutator.lastSet = thisIndex; - } + if(!mutator.fillEmpties(thisIndex)) return false; </#if> + boolean success = values.copyFromSafe(fromIndex, thisIndex, from); + success = success && bits.getMutator().setSafe(thisIndex, 1); return success; } public boolean copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){ - boolean success = bits.copyFromSafe(fromIndex, thisIndex, from.bits) && values.copyFromSafe(fromIndex, thisIndex, from.values); -<#if type.major == "VarLen"> - if (success) { - mutator.lastSet = thisIndex; - } -</#if> - return success; + <#if type.major == "VarLen"> + if(!mutator.fillEmpties(thisIndex)) return false; + </#if> + return bits.copyFromSafe(fromIndex, thisIndex, from.bits) && values.copyFromSafe(fromIndex, thisIndex, from.values); } @@ -408,13 +403,22 @@ public final class ${className} extends BaseValueVector implements <#if type.maj <#if type.major == "VarLen">lastSet = index;</#if> } + <#if type.major == "VarLen"> + private boolean fillEmpties(int index){ + for (int i = lastSet + 1; i < index; i++) { + if(!values.getMutator().setSafe(i, new byte[]{})) return false; + } + lastSet = index; + + return true; + } + </#if> + public boolean setSafe(int index, byte[] value, int start, int length) { <#if type.major != "VarLen"> throw new UnsupportedOperationException(); <#else> - for (int i = lastSet + 1; i < index; i++) { - if(!values.getMutator().setSafe(i, new byte[]{})) return false; - } + boolean b1 = bits.getMutator().setSafe(index, 1); boolean b2 = values.getMutator().setSafe(index, value, start, length); if(b1 && b2){ @@ -462,9 +466,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj public boolean setSafe(int index, Nullable${minor.class}Holder value) { <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - if(!values.getMutator().setSafe(i, new byte[]{})) return false; - } + if(!fillEmpties(index)) return false; </#if> boolean b1 = bits.getMutator().setSafe(index, 1); boolean b2 = values.getMutator().setSafe(index, value); @@ -481,9 +483,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj public boolean setSafe(int index, ${minor.class}Holder value) { <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - if(!values.getMutator().setSafe(i, new byte[]{})) return false; - } + if(!fillEmpties(index)) return false; </#if> boolean b1 = bits.getMutator().setSafe(index, 1); boolean b2 = values.getMutator().setSafe(index, value); @@ -500,15 +500,12 @@ public final class ${className} extends BaseValueVector implements <#if type.maj <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "TimeStampTZ" || minor.class == "Interval" || minor.class == "IntervalDay")> public boolean setSafe(int index, ${minor.javaType!type.javaType} value) { <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - values.getMutator().set(i, new byte[]{}); - } + if(!fillEmpties(index)) return false; </#if> boolean b1 = bits.getMutator().setSafe(index, 1); boolean b2 = values.getMutator().setSafe(index, value); if(b1 && b2){ setCount++; - <#if type.major == "VarLen">lastSet = index;</#if> return true; }else{ return false; @@ -520,9 +517,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj public void setValueCount(int valueCount) { assert valueCount >= 0; <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < valueCount; i++) { - values.getMutator().set(i, new byte[]{}); - } + fillEmpties(valueCount); </#if> Nullable${minor.class}Vector.this.valueCount = valueCount; values.getMutator().setValueCount(valueCount); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/85a2e042/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java index 88d858a..1f65761 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java @@ -102,6 +102,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea @Override public boolean allocateNewSafe() { if(!offsets.allocateNewSafe()) return false; + offsets.zeroVector(); if(vector != null){ return vector.allocateNewSafe(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/85a2e042/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java index 22471f0..bcf8ad7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java @@ -76,6 +76,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat public void allocateNew(int parentValueCount, int childValueCount) { clear(); offsets.allocateNew(parentValueCount+1); + offsets.zeroVector(); mutator.reset(); accessor.reset(); } @@ -187,6 +188,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat @Override public boolean allocateNewSafe() { if(!offsets.allocateNewSafe()) return false; + offsets.zeroVector(); for(ValueVector v : vectors.values()){ if(!v.allocateNewSafe()) return false; } @@ -216,7 +218,9 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat pairs = new TransferPair[vectors.size()]; int i =0; for(Map.Entry<String, ValueVector> e : vectors.entrySet()){ + int preSize = to.vectors.size(); ValueVector v = to.addOrGet(e.getKey(), e.getValue().getField().getType(), e.getValue().getClass()); + if(preSize != to.vectors.size()) v.allocateNew(); pairs[i++] = e.getValue().makeTransferPair(v); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/85a2e042/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java index 84195c3..0abdbd3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java @@ -21,22 +21,24 @@ import org.apache.drill.BaseTestQuery; import org.junit.Ignore; import org.junit.Test; -@Ignore("Test case fails intermittently, need to be fixed.") + public class JsonRecordReader2Test extends BaseTestQuery{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonRecordReader2Test.class); @Test public void testComplexJsonInput() throws Exception{ // test("select z[0]['orange'] from cp.`jsoninput/input2.json` limit 10"); - test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], z[0]['orange'], z[1]['pink'] from cp.`jsoninput/input2.json` "); - test("select x from cp.`jsoninput/input2.json`"); + test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], z[0]['orange'], z[1]['pink'] from cp.`jsoninput/input2.json` limit 10 "); +// test("select x from cp.`jsoninput/input2.json`"); // test("select z[0] from cp.`jsoninput/input2.json` limit 10"); } @Test public void z() throws Exception{ + for(int i =0 ; i < 5; i++){ test("select * from cp.`join/merge_join.json`"); + } } @Test http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/85a2e042/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 81054dd..1b42893 100644 --- a/pom.xml +++ b/pom.xml @@ -259,8 +259,8 @@ <artifactId>maven-surefire-plugin</artifactId> <version>2.17</version> <configuration> - <argLine>-Xms1g -Xmx2g -XX:MaxDirectMemorySize=13096M -XX:+CMSClassUnloadingEnabled</argLine> - <forkCount>1</forkCount> + <argLine>-Xms1g -Xmx2g -XX:MaxDirectMemorySize=2096M -XX:+CMSClassUnloadingEnabled</argLine> + <forkCount>4</forkCount> <reuseForks>true</reuseForks> <additionalClasspathElements> <additionalClasspathElement>./exec/jdbc/src/test/resources/storage-plugins.json</additionalClasspathElement>
