DRILL-192 Load fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/9d3e0a44 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/9d3e0a44 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/9d3e0a44 Branch: refs/heads/master Commit: 9d3e0a44f686967b4de6adc1ba86a1483bfdd803 Parents: 12634dc Author: witwolf <[email protected]> Authored: Sun Sep 1 17:07:42 2013 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Sun Sep 1 17:07:42 2013 -0700 ---------------------------------------------------------------------- .../codegen/templates/NullableValueVectors.java | 3 +- .../templates/VariableLengthVectors.java | 14 +-- .../drill/exec/record/vector/TestLoad.java | 111 +++++++++++++++++++ 3 files changed, 120 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9d3e0a44/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java index bf28ecc..f528ce1 100644 --- a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java +++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java @@ -83,6 +83,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj // remove bits part of buffer. buf = buf.slice(loaded, buf.capacity() - loaded); + dataBytes -= loaded; loaded += values.load(dataBytes, valueCount, buf); return loaded; } @@ -90,7 +91,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj @Override public void load(FieldMetadata metadata, ByteBuf buffer) { assert this.field.getDef().equals(metadata.getDef()); - int loaded = load(metadata.getVarByteLength(), metadata.getValueCount(), buffer); + int loaded = load(metadata.getBufferLength(), metadata.getValueCount(), buffer); assert metadata.getBufferLength() == loaded; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9d3e0a44/sandbox/prototype/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java index 3e5464c..d132eb0 100644 --- a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java +++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java @@ -43,7 +43,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } public int getValueCapacity(){ - return offsetVector.getValueCapacity(); + return offsetVector.getValueCapacity() - 1; } public int getByteCapacity(){ @@ -60,27 +60,26 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V @Override public FieldMetadata getMetadata() { - int len = (valueCount + 1) * ${type.width} + getVarByteLength(); return FieldMetadata.newBuilder() .setDef(getField().getDef()) .setValueCount(valueCount) .setVarByteLength(getVarByteLength()) - .setBufferLength(len) + .setBufferLength(getBufferSize()) .build(); } public int load(int dataBytes, int valueCount, ByteBuf buf){ this.valueCount = valueCount; int loaded = offsetVector.load(valueCount+1, buf); - data = buf.slice(loaded, dataBytes); + data = buf.slice(loaded, dataBytes - loaded); data.retain(); - return loaded + dataBytes; + return dataBytes; } @Override public void load(FieldMetadata metadata, ByteBuf buffer) { assert this.field.getDef().equals(metadata.getDef()); - int loaded = load(metadata.getVarByteLength(), metadata.getValueCount(), buffer); + int loaded = load(metadata.getBufferLength(), metadata.getValueCount(), buffer); assert metadata.getBufferLength() == loaded; } @@ -168,6 +167,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V data = allocator.buffer(totalBytes); data.readerIndex(0); offsetVector.allocateNew(valueCount+1); + offsetVector.getMutator().set(0,0); } public Accessor getAccessor(){ @@ -307,7 +307,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V @Override public void generateTestData(){ boolean even = true; - for(int i =0; i < valueCount; i++, even = !even){ + for(int i =0; i < getValueCapacity(); i++, even = !even){ if(even){ set(i, new String("aaaaa").getBytes(Charsets.UTF_8)); }else{ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9d3e0a44/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java ---------------------------------------------------------------------- diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java new file mode 100644 index 0000000..70f43ab --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java @@ -0,0 +1,111 @@ +package org.apache.drill.exec.record.vector; + +import com.google.common.collect.Lists; +import io.netty.buffer.ByteBuf; +import org.apache.drill.common.expression.ExpressionPosition; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.TypeProtos.*; +import org.apache.drill.common.types.Types; +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.RecordBatchLoader; +import org.apache.drill.exec.record.VectorWrapper; +import org.apache.drill.exec.record.WritableBatch; +import org.apache.drill.exec.vector.*; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Created with IntelliJ IDEA. + * User: witwolf + * Date: 8/30/13 + * Time: 5:00 PM + */ + +public class TestLoad { + @Test + public void testLoadValueVector() { + BufferAllocator allocator = BufferAllocator.getAllocator(null); + ValueVector fixedV = new IntVector( + MaterializedField.create(new SchemaPath("ints", ExpressionPosition.UNKNOWN), Types.required(MinorType.INT)), + allocator); + ValueVector varlenV = new VarCharVector( + MaterializedField.create(new SchemaPath("chars", ExpressionPosition.UNKNOWN), Types.required(MinorType.VARCHAR)), + allocator + ); + ValueVector nullableVarlenV = new NullableVarCharVector( + MaterializedField.create(new SchemaPath("chars", ExpressionPosition.UNKNOWN), Types.optional(MinorType.VARCHAR)), + allocator + ); + + List<ValueVector> vectors = Lists.newArrayList(fixedV, varlenV, nullableVarlenV); + for (ValueVector v : vectors) { + AllocationHelper.allocate(v, 100, 50); + v.getMutator().generateTestData(); + v.getMutator().setValueCount(100); + } + + WritableBatch writableBatch = WritableBatch.getBatchNoSV(100, vectors); + RecordBatchLoader batchLoader = new RecordBatchLoader(allocator); + ByteBuf[] byteBufs = writableBatch.getBuffers(); + int bytes = 0; + for (int i = 0; i < byteBufs.length; i++) { + bytes += byteBufs[i].writerIndex(); + } + ByteBuf byteBuf = allocator.buffer(bytes); + int index = 0; + for (int i = 0; i < byteBufs.length; i++) { + byteBufs[i].readBytes(byteBuf, index, byteBufs[i].writerIndex()); + index += byteBufs[i].writerIndex(); + } + byteBuf.writerIndex(bytes); + try { + batchLoader.load(writableBatch.getDef(), byteBuf); + boolean firstColumn = true; + int recordCount = 0; + for (VectorWrapper<?> v : batchLoader) { + if (firstColumn) { + firstColumn = false; + } else { + System.out.print("\t"); + } + System.out.print(v.getField().getName()); + System.out.print("["); + System.out.print(v.getField().getType().getMinorType()); + System.out.print("]"); + } + + System.out.println(); + for (int r = 0; r < batchLoader.getRecordCount(); r++) { + boolean first = true; + recordCount++; + for (VectorWrapper<?> v : batchLoader) { + if (first) { + first = false; + } else { + System.out.print("\t"); + } + ValueVector.Accessor accessor = v.getValueVector().getAccessor(); + if (v.getField().getType().getMinorType() == TypeProtos.MinorType.VARCHAR) { + Object obj = accessor.getObject(r) ; + if(obj != null) + System.out.print(new String((byte[]) accessor.getObject(r))); + else + System.out.print("NULL"); + } else { + System.out.print(accessor.getObject(r)); + } + } + if (!first) System.out.println(); + } + assertEquals(100, recordCount); + } catch (Exception e) { + e.printStackTrace(); + } + } + +}
