Repository: drill Updated Branches: refs/heads/master b55e2328d -> 718abac46
http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java index 6431d88..2f5ec18 100644 --- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java @@ -35,27 +35,24 @@ package org.apache.drill.exec.vector; <#include "/@includes/vv_imports.ftl" /> - -@SuppressWarnings("unused") /** * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or * repeated protobuf field). The implementation uses two additional value vectors; one to convert * the index offset to the underlying element offset, and another to store the number of values * in the vector. * - * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker. + * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector implements Repeated<#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>VectorLike { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Repeated${minor.class}Vector.class); + //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Repeated${minor.class}Vector.class); // we maintain local reference to concrete vector type for performance reasons. private ${minor.class}Vector values; private final FieldReader reader = new Repeated${minor.class}ReaderImpl(Repeated${minor.class}Vector.this); private final Mutator mutator = new Mutator(); private final Accessor accessor = new Accessor(); - - + public Repeated${minor.class}Vector(MaterializedField field, BufferAllocator allocator) { super(field, allocator); addOrGetVector(VectorDescriptor.create(Types.required(field.getType().getMinorType()))); @@ -72,17 +69,17 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector } @Override - public FieldReader getReader(){ + public FieldReader getReader() { return reader; } @Override - public ${minor.class}Vector getDataVector(){ + public ${minor.class}Vector getDataVector() { return values; } @Override - public TransferPair getTransferPair(){ + public TransferPair getTransferPair() { return new TransferImpl(getField()); } @@ -105,7 +102,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector return result; } - public void transferTo(Repeated${minor.class}Vector target){ + public void transferTo(Repeated${minor.class}Vector target) { target.clear(); offsets.transferTo(target.offsets); values.transferTo(target.values); @@ -115,11 +112,11 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector public void splitAndTransferTo(final int startIndex, final int groups, Repeated${minor.class}Vector to) { final UInt4Vector.Accessor a = offsets.getAccessor(); final UInt4Vector.Mutator m = to.offsets.getMutator(); - - final int startPos = offsets.getAccessor().get(startIndex); - final int endPos = offsets.getAccessor().get(startIndex + groups); + + final int startPos = a.get(startIndex); + final int endPos = a.get(startIndex + groups); final int valuesToCopy = endPos - startPos; - + values.splitAndTransferTo(startPos, valuesToCopy, to.values); to.offsets.clear(); to.offsets.allocateNew(groups + 1); @@ -130,53 +127,57 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector } m.setValueCount(groups == 0 ? 0 : groups + 1); } - - private class TransferImpl implements TransferPair{ - Repeated${minor.class}Vector to; - - public TransferImpl(MaterializedField field){ + + private class TransferImpl implements TransferPair { + final Repeated${minor.class}Vector to; + + public TransferImpl(MaterializedField field) { this.to = new Repeated${minor.class}Vector(field, allocator); } - public TransferImpl(Repeated${minor.class}Vector to){ + public TransferImpl(Repeated${minor.class}Vector to) { this.to = to; } - public Repeated${minor.class}Vector getTo(){ + @Override + public Repeated${minor.class}Vector getTo() { return to; } - - public void transfer(){ + + @Override + public void transfer() { transferTo(to); } + @Override public void splitAndTransfer(int startIndex, int length) { splitAndTransferTo(startIndex, length, to); } - + @Override public void copyValueSafe(int fromIndex, int toIndex) { to.copyFromSafe(fromIndex, toIndex, Repeated${minor.class}Vector.this); } } - public void copyFrom(int inIndex, int outIndex, Repeated${minor.class}Vector v){ - final int count = v.getAccessor().getInnerValueCountAt(inIndex); - getMutator().startNewValue(outIndex); + public void copyFrom(int inIndex, int outIndex, Repeated${minor.class}Vector v) { + final Accessor vAccessor = v.getAccessor(); + final int count = vAccessor.getInnerValueCountAt(inIndex); + mutator.startNewValue(outIndex); for (int i = 0; i < count; i++) { - getMutator().add(outIndex, v.getAccessor().get(inIndex, i)); + mutator.add(outIndex, vAccessor.get(inIndex, i)); } } - public void copyFromSafe(int inIndex, int outIndex, Repeated${minor.class}Vector v){ - final int count = v.getAccessor().getInnerValueCountAt(inIndex); - getMutator().startNewValue(outIndex); + public void copyFromSafe(int inIndex, int outIndex, Repeated${minor.class}Vector v) { + final Accessor vAccessor = v.getAccessor(); + final int count = vAccessor.getInnerValueCountAt(inIndex); + mutator.startNewValue(outIndex); for (int i = 0; i < count; i++) { - getMutator().addSafe(outIndex, v.getAccessor().get(inIndex, i)); + mutator.addSafe(outIndex, vAccessor.get(inIndex, i)); } } - public boolean allocateNewSafe() { /* boolean to keep track if all the memory allocation were successful * Used in the case of composite vectors when we need to allocate multiple @@ -197,7 +198,8 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector mutator.reset(); return true; } - + + @Override public void allocateNew() { try { offsets.allocateNew(); @@ -216,10 +218,10 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector return super.getMetadataBuilder() .setVarByteLength(values.getVarByteLength()); } - + public void allocateNew(int totalBytes, int valueCount, int innerValueCount) { try { - offsets.allocateNew(valueCount+1); + offsets.allocateNew(valueCount + 1); values.allocateNew(totalBytes, innerValueCount); } catch (OutOfMemoryRuntimeException e) { clear(); @@ -235,6 +237,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector <#else> + @Override public void allocateNew(int valueCount, int innerValueCount) { clear(); /* boolean to keep track if all the memory allocation were successful @@ -244,7 +247,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector */ boolean success = false; try { - offsets.allocateNew(valueCount+1); + offsets.allocateNew(valueCount + 1); values.allocateNew(innerValueCount); } catch(OutOfMemoryRuntimeException e){ clear(); @@ -253,27 +256,29 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector offsets.zeroVector(); mutator.reset(); } - </#if> + </#if> // This is declared a subclass of the accessor declared inside of FixedWidthVector, this is also used for // variable length vectors, as they should ahve consistent interface as much as possible, if they need to diverge // in the future, the interface shold be declared in the respective value vector superclasses for fixed and variable // and we should refer to each in the generation template public final class Accessor extends BaseRepeatedValueVector.BaseRepeatedAccessor { - + @Override public List<${friendlyType}> getObject(int index) { - List<${friendlyType}> vals = new JsonStringArrayList(); - int start = offsets.getAccessor().get(index); - int end = offsets.getAccessor().get(index+1); - for(int i = start; i < end; i++){ - vals.add(values.getAccessor().getObject(i)); + final List<${friendlyType}> vals = new JsonStringArrayList<>(); + final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor(); + final int start = offsetsAccessor.get(index); + final int end = offsetsAccessor.get(index + 1); + final ${minor.class}Vector.Accessor valuesAccessor = values.getAccessor(); + for(int i = start; i < end; i++) { + vals.add(valuesAccessor.getObject(i)); } return vals; } - - public ${friendlyType} getSingleObject(int index, int arrayIndex){ - int start = offsets.getAccessor().get(index); + + public ${friendlyType} getSingleObject(int index, int arrayIndex) { + final int start = offsets.getAccessor().get(index); return values.getAccessor().getObject(start + arrayIndex); } @@ -291,21 +296,21 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex); } - public void get(int index, Repeated${minor.class}Holder holder){ + public void get(int index, Repeated${minor.class}Holder holder) { holder.start = offsets.getAccessor().get(index); holder.end = offsets.getAccessor().get(index+1); holder.vector = values; } public void get(int index, int positionIndex, ${minor.class}Holder holder) { - int offset = offsets.getAccessor().get(index); + final int offset = offsets.getAccessor().get(index); assert offset >= 0; assert positionIndex < getInnerValueCountAt(index); values.getAccessor().get(offset + positionIndex, holder); } - + public void get(int index, int positionIndex, Nullable${minor.class}Holder holder) { - int offset = offsets.getAccessor().get(index); + final int offset = offsets.getAccessor().get(index); assert offset >= 0; if (positionIndex >= getInnerValueCountAt(index)) { holder.isSet = 0; @@ -314,10 +319,9 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector values.getAccessor().get(offset + positionIndex, holder); } } - - public final class Mutator extends BaseRepeatedValueVector.BaseRepeatedMutator implements RepeatedMutator { - private Mutator() { } + public final class Mutator extends BaseRepeatedValueVector.BaseRepeatedMutator implements RepeatedMutator { + private Mutator() {} /** * Add an element to the given record index. This is similar to the set() method in other @@ -338,7 +342,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector } public void addSafe(int index, byte[] bytes, int start, int length) { - int nextOffset = offsets.getAccessor().get(index+1); + final int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().setSafe(nextOffset, bytes, start, length); offsets.getMutator().setSafe(index+1, nextOffset+1); } @@ -346,76 +350,77 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector <#else> public void addSafe(int index, ${minor.javaType!type.javaType} srcValue) { - int nextOffset = offsets.getAccessor().get(index+1); + final int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().setSafe(nextOffset, srcValue); offsets.getMutator().setSafe(index+1, nextOffset+1); } - + </#if> - - public void setSafe(int index, Repeated${minor.class}Holder h){ - ${minor.class}Holder ih = new ${minor.class}Holder(); - getMutator().startNewValue(index); + public void setSafe(int index, Repeated${minor.class}Holder h) { + final ${minor.class}Holder ih = new ${minor.class}Holder(); + final ${minor.class}Vector.Accessor hVectorAccessor = h.vector.getAccessor(); + mutator.startNewValue(index); for(int i = h.start; i < h.end; i++){ - h.vector.getAccessor().get(i, ih); - getMutator().addSafe(index, ih); + hVectorAccessor.get(i, ih); + mutator.addSafe(index, ih); } } - - public void addSafe(int index, ${minor.class}Holder holder){ + + public void addSafe(int index, ${minor.class}Holder holder) { int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().setSafe(nextOffset, holder); offsets.getMutator().setSafe(index+1, nextOffset+1); } - - public void addSafe(int index, Nullable${minor.class}Holder holder){ - int nextOffset = offsets.getAccessor().get(index+1); + + public void addSafe(int index, Nullable${minor.class}Holder holder) { + final int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().setSafe(nextOffset, holder); offsets.getMutator().setSafe(index+1, nextOffset+1); } - + <#if (fields?size > 1) && !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")> - public void addSafe(int arrayIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>){ + public void addSafe(int arrayIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) { int nextOffset = offsets.getAccessor().get(arrayIndex+1); values.getMutator().setSafe(nextOffset, <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>); offsets.getMutator().setSafe(arrayIndex+1, nextOffset+1); } </#if> - - protected void add(int index, ${minor.class}Holder holder){ + + protected void add(int index, ${minor.class}Holder holder) { int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().set(nextOffset, holder); offsets.getMutator().set(index+1, nextOffset+1); } - - public void add(int index, Repeated${minor.class}Holder holder){ - + + public void add(int index, Repeated${minor.class}Holder holder) { + ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor(); ${minor.class}Holder innerHolder = new ${minor.class}Holder(); - - for(int i = holder.start; i < holder.end; i++){ + + for(int i = holder.start; i < holder.end; i++) { accessor.get(i, innerHolder); add(index, innerHolder); } } - public void generateTestData(final int valCount){ - int[] sizes = {1,2,0,6}; + @Override + public void generateTestData(final int valCount) { + final int[] sizes = {1, 2, 0, 6}; int size = 0; int runningOffset = 0; - for(int i =1; i < valCount+1; i++, size++){ + final UInt4Vector.Mutator offsetsMutator = offsets.getMutator(); + for(int i = 1; i < valCount + 1; i++, size++) { runningOffset += sizes[size % sizes.length]; - offsets.getMutator().set(i, runningOffset); + offsetsMutator.set(i, runningOffset); } - values.getMutator().generateTestData(valCount*9); + values.getMutator().generateTestData(valCount * 9); setValueCount(size); } - public void reset(){ - + @Override + public void reset() { } - } } </#list> http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java b/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java index c175900..6d92a2f 100644 --- a/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java +++ b/exec/java-exec/src/main/codegen/templates/StringOutputRecordWriter.java @@ -30,7 +30,6 @@ package org.apache.drill.exec.store; import com.google.common.collect.Lists; import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.expr.holders.*; -import org.apache.drill.exec.memory.TopLevelAllocator; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.VectorAccessible; @@ -50,6 +49,8 @@ import java.util.Map; * to output the data in string format instead of implementing addField for each type holder. * * This is useful for text format writers such as CSV, TSV etc. + * + * NB: Source code generated using FreeMarker template ${.template_name} */ public abstract class StringOutputRecordWriter extends AbstractRecordWriter { http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java index fc08ac6..ac4d539 100644 --- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java +++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java @@ -29,7 +29,6 @@ import org.apache.drill.exec.vector.VariableWidthVector; <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> - <#if type.major == "VarLen"> <@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" /> @@ -47,11 +46,10 @@ package org.apache.drill.exec.vector; * The width of each element is ${type.width} byte(s) * The equivalent Java primitive is '${minor.javaType!type.javaType}' * - * Source code generated using FreeMarker template ${.template_name} + * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -@SuppressWarnings("unused") -public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); +public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); private static final int DEFAULT_RECORD_BYTE_COUNT = 8; private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT; @@ -68,7 +66,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V private final UInt${type.width}Vector.Accessor oAccessor; - private int allocationSizeInBytes = INITIAL_BYTE_COUNT; private int allocationMonitor = 0; @@ -81,27 +78,29 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } @Override - public FieldReader getReader(){ + public FieldReader getReader() { return reader; } - public int getBufferSize(){ - if (getAccessor().getValueCount() == 0) return 0; + @Override + public int getBufferSize() { + if (getAccessor().getValueCount() == 0) { + return 0; + } return offsetVector.getBufferSize() + data.writerIndex(); } - int getSizeFromCount(int valueCount) { - return valueCount * ${type.width}; - } - - public int getValueCapacity(){ + @Override + public int getValueCapacity() { return offsetVector.getValueCapacity() - 1; } - public int getByteCapacity(){ + @Override + public int getByteCapacity() { return data.capacity(); } + @Override public int getCurrentSizeInBytes() { return offsetVector.getAccessor().get(getAccessor().getValueCount()); } @@ -110,31 +109,34 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V * Return the number of bytes contained in the current var len byte vector. * @return */ - public int getVarByteLength(){ + public int getVarByteLength() { final int valueCount = getAccessor().getValueCount(); - if(valueCount == 0) return 0; + if(valueCount == 0) { + return 0; + } return offsetVector.getAccessor().get(valueCount); } @Override public SerializedField getMetadata() { - return getMetadataBuilder() // - .setValueCount(getAccessor().getValueCount()) // - .setVarByteLength(getVarByteLength()) // - .setBufferLength(getBufferSize()) // + return getMetadataBuilder() + .setValueCount(getAccessor().getValueCount()) + .setVarByteLength(getVarByteLength()) + .setBufferLength(getBufferSize()) .build(); } - public int load(int dataBytes, int valueCount, DrillBuf buf){ - if(valueCount == 0){ - allocateNew(0,0); + @Override + public int load(int dataBytes, int valueCount, DrillBuf buf) { + if(valueCount == 0) { + allocateNew(0, 0); return 0; } clear(); - int loaded = offsetVector.load(valueCount+1, buf); + final int loaded = offsetVector.load(valueCount + 1, buf); data = buf.slice(loaded, dataBytes - loaded); - data.retain(); - return dataBytes; + data.retain(1); + return dataBytes; } @Override @@ -150,109 +152,115 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V offsetVector.clear(); } - @Override public DrillBuf[] getBuffers(boolean clear) { - DrillBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(false), super.getBuffers(false), DrillBuf.class); + final DrillBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(false), super.getBuffers(false), DrillBuf.class); if (clear) { // does not make much sense but we have to retain buffers even when clear is set. refactor this interface. - for (DrillBuf buffer:buffers) { - buffer.retain(); + for (final DrillBuf buffer:buffers) { + buffer.retain(1); } clear(); } return buffers; } - public long getOffsetAddr(){ + public long getOffsetAddr() { return offsetVector.getBuffer().memoryAddress(); } - public UInt${type.width}Vector getOffsetVector(){ + public UInt${type.width}Vector getOffsetVector() { return offsetVector; } - public TransferPair getTransferPair(){ + @Override + public TransferPair getTransferPair() { return new TransferImpl(getField()); } + + @Override public TransferPair getTransferPair(FieldReference ref){ return new TransferImpl(getField().withPath(ref)); } + @Override public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((${minor.class}Vector) to); } - public void transferTo(${minor.class}Vector target){ + public void transferTo(${minor.class}Vector target) { target.clear(); this.offsetVector.transferTo(target.offsetVector); target.data = data; - target.data.retain(); + target.data.retain(1); clear(); } public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) { - int startPoint = this.offsetVector.getAccessor().get(startIndex); - int sliceLength = this.offsetVector.getAccessor().get(startIndex + length) - startPoint; - target.offsetVector.clear(); + UInt${type.width}Vector.Accessor offsetVectorAccessor = this.offsetVector.getAccessor(); + final int startPoint = offsetVectorAccessor.get(startIndex); + final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint; + target.clear(); target.offsetVector.allocateNew(length + 1); + offsetVectorAccessor = this.offsetVector.getAccessor(); + final UInt4Vector.Mutator offsetVectorMutator = target.offsetVector.getMutator(); for (int i = 0; i < length + 1; i++) { - target.offsetVector.getMutator().set(i, this.offsetVector.getAccessor().get(startIndex + i) - startPoint); + offsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint); } - target.data = this.data.slice(startPoint, sliceLength); - target.data.retain(); + target.data = data.slice(startPoint, sliceLength); + target.data.retain(1); target.getMutator().setValueCount(length); } - protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ - int start = from.offsetVector.getAccessor().get(fromIndex); - int end = from.offsetVector.getAccessor().get(fromIndex+1); - int len = end - start; + protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) { + final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor(); + final int start = fromOffsetVectorAccessor.get(fromIndex); + final int end = fromOffsetVectorAccessor.get(fromIndex + 1); + final int len = end - start; - int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}); + final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}); from.data.getBytes(start, data, outputStart, len); offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len); } - public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ - - int start = from.offsetVector.getAccessor().get(fromIndex); - int end = from.offsetVector.getAccessor().get(fromIndex+1); - int len = end - start; - - int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}); + public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) { + final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor(); + final int start = fromOffsetVectorAccessor.get(fromIndex); + final int end = fromOffsetVectorAccessor.get(fromIndex + 1); + final int len = end - start; + final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}); while(data.capacity() < outputStart + len) { - reAlloc(); + reAlloc(); } offsetVector.getMutator().setSafe(thisIndex + 1, outputStart + len); - from.data.getBytes(start, data, outputStart, len); - return true; } - - private class TransferImpl implements TransferPair{ + private class TransferImpl implements TransferPair { ${minor.class}Vector to; - public TransferImpl(MaterializedField field){ - this.to = new ${minor.class}Vector(field, allocator); + public TransferImpl(MaterializedField field) { + to = new ${minor.class}Vector(field, allocator); } - public TransferImpl(${minor.class}Vector to){ + public TransferImpl(${minor.class}Vector to) { this.to = to; } - public ${minor.class}Vector getTo(){ + @Override + public ${minor.class}Vector getTo() { return to; } - public void transfer(){ + @Override + public void transfer() { transferTo(to); } + @Override public void splitAndTransfer(int startIndex, int length) { splitAndTransferTo(startIndex, length, to); } @@ -273,8 +281,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V offsetVector.setInitialCapacity(valueCount + 1); } + @Override public void allocateNew() { - if(!allocateNewSafe()){ + if(!allocateNewSafe()) { throw new OutOfMemoryRuntimeException("Failure while allocating buffer."); } } @@ -301,7 +310,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V * clear all the memory that we allocated */ try { - final int requestedSize = (int)curAllocationSize; + final int requestedSize = (int) curAllocationSize; data = allocator.buffer(requestedSize); offsetVector.allocateNew(); } catch (OutOfMemoryRuntimeException e) { @@ -313,6 +322,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V return true; } + @Override public void allocateNew(int totalBytes, int valueCount) { clear(); assert totalBytes >= 0; @@ -352,42 +362,45 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V ++allocationMonitor; } - public Accessor getAccessor(){ + @Override + public Accessor getAccessor() { return accessor; } + @Override public Mutator getMutator() { return mutator; } public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor { final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor(); - - public long getStartEnd(int index){ + public long getStartEnd(int index) { return oAccessor.getTwoAsLong(index); } public byte[] get(int index) { assert index >= 0; - int startIdx = oAccessor.get(index); - int length = oAccessor.get(index + 1) - startIdx; + final int startIdx = oAccessor.get(index); + final int length = oAccessor.get(index + 1) - startIdx; assert length >= 0; - byte[] dst = new byte[length]; + final byte[] dst = new byte[length]; data.getBytes(startIdx, dst, 0, length); return dst; } + @Override public int getValueLength(int index) { - return offsetVector.getAccessor().get(index + 1) - offsetVector.getAccessor().get(index); + final UInt${type.width}Vector.Accessor offsetVectorAccessor = offsetVector.getAccessor(); + return offsetVectorAccessor.get(index + 1) - offsetVectorAccessor.get(index); } - public void get(int index, ${minor.class}Holder holder){ + public void get(int index, ${minor.class}Holder holder) { holder.start = oAccessor.get(index); holder.end = oAccessor.get(index + 1); holder.buffer = data; } - public void get(int index, Nullable${minor.class}Holder holder){ + public void get(int index, Nullable${minor.class}Holder holder) { holder.isSet = 1; holder.start = oAccessor.get(index); holder.end = oAccessor.get(index + 1); @@ -397,6 +410,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V <#switch minor.class> <#case "VarChar"> + @Override public ${friendlyType} getObject(int index) { Text text = new Text(); text.set(get(index)); @@ -404,28 +418,29 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } <#break> <#case "Var16Char"> + @Override public ${friendlyType} getObject(int index) { return new String(get(index), Charsets.UTF_16); } <#break> <#default> + @Override public ${friendlyType} getObject(int index) { return get(index); } - </#switch> - - + @Override public int getValueCount() { return Math.max(offsetVector.getAccessor().getValueCount()-1, 0); } - public boolean isNull(int index){ + @Override + public boolean isNull(int index) { return false; } - public UInt${type.width}Vector getOffsetVector(){ + public UInt${type.width}Vector getOffsetVector() { return offsetVector; } } @@ -450,7 +465,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V */ protected void set(int index, byte[] bytes) { assert index >= 0; - int currentOffset = offsetVector.getAccessor().get(index); + final int currentOffset = offsetVector.getAccessor().get(index); offsetVector.getMutator().set(index + 1, currentOffset + bytes.length); data.setBytes(currentOffset, bytes, 0, bytes.length); } @@ -458,12 +473,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V public void setSafe(int index, byte[] bytes) { assert index >= 0; - int currentOffset = offsetVector.getAccessor().get(index); - while (data.capacity() < currentOffset + bytes.length) { + final int currentOffset = offsetVector.getAccessor().get(index); + while(data.capacity() < currentOffset + bytes.length) { reAlloc(); } offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length); - offsetVector.getMutator().set(index + 1, currentOffset + bytes.length); data.setBytes(currentOffset, bytes, 0, bytes.length); } @@ -477,7 +491,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V */ protected void set(int index, byte[] bytes, int start, int length) { assert index >= 0; - int currentOffset = offsetVector.getAccessor().get(index); + final int currentOffset = offsetVector.getAccessor().get(index); offsetVector.getMutator().set(index + 1, currentOffset + length); data.setBytes(currentOffset, bytes, start, length); } @@ -497,7 +511,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V public void setSafe(int index, byte[] bytes, int start, int length) { assert index >= 0; - int currentOffset = offsetVector.getAccessor().get(index); + final int currentOffset = offsetVector.getAccessor().get(index); while (data.capacity() < currentOffset + length) { reAlloc(); @@ -506,8 +520,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V data.setBytes(currentOffset, bytes, start, length); } + @Override public void setValueLengthSafe(int index, int length) { - int offset = offsetVector.getAccessor().get(index); + final int offset = offsetVector.getAccessor().get(index); while(data.capacity() < offset + length ) { reAlloc(); } @@ -515,10 +530,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } - public void setSafe(int index, int start, int end, DrillBuf buffer){ - int len = end - start; - - int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); + public void setSafe(int index, int start, int end, DrillBuf buffer) { + final int len = end - start; + final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); while(data.capacity() < outputStart + len) { reAlloc(); @@ -528,13 +542,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V buffer.getBytes(start, data, outputStart, len); } - - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setSafe(int index, Nullable${minor.class}Holder holder) { assert holder.isSet == 1; - int start = holder.start; - int end = holder.end; - int len = end - start; + final int start = holder.start; + final int end = holder.end; + final int len = end - start; int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); @@ -546,13 +559,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V offsetVector.getMutator().setSafe( index+1, outputStart + len); } - public void setSafe(int index, ${minor.class}Holder holder){ - - int start = holder.start; - int end = holder.end; - int len = end - start; - - int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); + public void setSafe(int index, ${minor.class}Holder holder) { + final int start = holder.start; + final int end = holder.end; + final int len = end - start; + final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); while(data.capacity() < outputStart + len) { reAlloc(); @@ -562,31 +573,32 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V offsetVector.getMutator().setSafe( index+1, outputStart + len); } - protected void set(int index, int start, int length, DrillBuf buffer){ + protected void set(int index, int start, int length, DrillBuf buffer) { assert index >= 0; - int currentOffset = offsetVector.getAccessor().get(index); + final int currentOffset = offsetVector.getAccessor().get(index); offsetVector.getMutator().set(index + 1, currentOffset + length); - DrillBuf bb = buffer.slice(start, length); + final DrillBuf bb = buffer.slice(start, length); data.setBytes(currentOffset, bb); } - protected void set(int index, Nullable${minor.class}Holder holder){ - int length = holder.end - holder.start; - int currentOffset = offsetVector.getAccessor().get(index); + protected void set(int index, Nullable${minor.class}Holder holder) { + final int length = holder.end - holder.start; + final int currentOffset = offsetVector.getAccessor().get(index); offsetVector.getMutator().set(index + 1, currentOffset + length); data.setBytes(currentOffset, holder.buffer, holder.start, length); } - protected void set(int index, ${minor.class}Holder holder){ - int length = holder.end - holder.start; - int currentOffset = offsetVector.getAccessor().get(index); + protected void set(int index, ${minor.class}Holder holder) { + final int length = holder.end - holder.start; + final int currentOffset = offsetVector.getAccessor().get(index); offsetVector.getMutator().set(index + 1, currentOffset + length); data.setBytes(currentOffset, holder.buffer, holder.start, length); } + @Override public void setValueCount(int valueCount) { - int currentByteCapacity = getByteCapacity(); - int idx = offsetVector.getAccessor().get(valueCount); + final int currentByteCapacity = getByteCapacity(); + final int idx = offsetVector.getAccessor().get(valueCount); data.writerIndex(idx); if (valueCount > 0 && currentByteCapacity > idx * 2) { incrementAllocationMonitor(); @@ -598,30 +610,26 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } @Override - public void generateTestData(int size){ + public void generateTestData(int size) { boolean even = true; <#switch minor.class> <#case "Var16Char"> - java.nio.charset.Charset charset = Charsets.UTF_16; + final java.nio.charset.Charset charset = Charsets.UTF_16; <#break> <#case "VarChar"> <#default> - java.nio.charset.Charset charset = Charsets.UTF_8; + final java.nio.charset.Charset charset = Charsets.UTF_8; </#switch> - for(int i =0; i < size; i++, even = !even){ - if(even){ - set(i, new String("aaaaa").getBytes(charset)); - }else{ - set(i, new String("bbbbbbbbbb").getBytes(charset)); - } + final byte[] evenValue = new String("aaaaa").getBytes(charset); + final byte[] oddValue = new String("bbbbbbbbbb").getBytes(charset); + for(int i =0; i < size; i++, even = !even) { + set(i, even ? evenValue : oddValue); } setValueCount(size); } } - } - </#if> <#-- type.major --> </#list> </#list> http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java index 579eed6..42e0972 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java @@ -24,21 +24,35 @@ import org.apache.drill.exec.record.MaterializedField; public abstract class BaseDataValueVector extends BaseValueVector { + protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this + protected DrillBuf data; public BaseDataValueVector(MaterializedField field, BufferAllocator allocator) { super(field, allocator); - this.data = allocator.getEmpty(); + data = allocator.getEmpty(); } @Override public void clear() { - data.release(); + if (data != null) { + data.release(); + } data = allocator.getEmpty(); super.clear(); } @Override + public void close() { + clear(); + if (data != null) { + data.release(); + data = null; + } + super.close(); + } + + @Override public DrillBuf[] getBuffers(boolean clear) { DrillBuf[] out; if (getBufferSize() == 0) { @@ -47,7 +61,7 @@ public abstract class BaseDataValueVector extends BaseValueVector { out = new DrillBuf[]{data}; if (clear) { data.readerIndex(0); - data.retain(); + data.retain(1); } } if (clear) { @@ -56,6 +70,7 @@ public abstract class BaseDataValueVector extends BaseValueVector { return out; } + @Override public int getBufferSize() { if (getAccessor().getValueCount() == 0) { return 0; http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java index a4a35e2..88a56f8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/ComplexWriterImpl.java @@ -29,12 +29,12 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; import com.google.common.base.Preconditions; -public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class); +public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter { +// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class); - SingleMapWriter mapRoot; - SingleListWriter listRoot; - MapVector container; + private SingleMapWriter mapRoot; + private SingleListWriter listRoot; + private final MapVector container; Mode mode = Mode.INIT; private final String name; @@ -47,6 +47,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri this.container = container; } + @Override public MaterializedField getField() { return container.getField(); } @@ -60,10 +61,21 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri StateTool.check(mode, modes); } + @Override public void reset(){ setPosition(0); } + @Override + public void close() throws Exception { + clear(); + mapRoot.close(); + if (listRoot != null) { + listRoot.close(); + } + } + + @Override public void clear(){ switch(mode){ case MAP: @@ -75,6 +87,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri } } + @Override public void setValueCount(int count){ switch(mode){ case MAP: @@ -146,9 +159,9 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri @Override public void allocate() { - if(mapRoot != null){ + if(mapRoot != null) { mapRoot.allocate(); - }else if(listRoot != null){ + } else if(listRoot != null) { listRoot.allocate(); } } @@ -184,7 +197,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri @Override public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) { - ValueVector v = vc.addOrGet(name, type, clazz); + final ValueVector v = vc.addOrGet(name, type, clazz); putChild(name, v); return this.typeify(v, clazz); http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java index 5aea0ca..95c651f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java @@ -27,19 +27,20 @@ import org.apache.drill.exec.vector.complex.MapVector; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; public class VectorContainerWriter extends AbstractFieldWriter implements ComplexWriter { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorContainerWriter.class); + //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorContainerWriter.class); - SingleMapWriter mapRoot; - SpecialMapVector mapVector; - OutputMutator mutator; + private final SingleMapWriter mapRoot; + private final SpecialMapVector mapVector; + private final OutputMutator mutator; public VectorContainerWriter(OutputMutator mutator) { super(null); this.mutator = mutator; - this.mapVector = new SpecialMapVector(mutator.getCallBack()); - this.mapRoot = new SingleMapWriter(mapVector, this); + mapVector = new SpecialMapVector(mutator.getCallBack()); + mapRoot = new SingleMapWriter(mapVector, this); } + @Override public MaterializedField getField() { return mapVector.getField(); } @@ -53,10 +54,19 @@ public class VectorContainerWriter extends AbstractFieldWriter implements Comple return mapVector; } + @Override public void reset() { setPosition(0); } + @Override + public void close() throws Exception { + clear(); + mapRoot.close(); + mapVector.close(); + } + + @Override public void clear() { mapRoot.clear(); } @@ -65,6 +75,7 @@ public class VectorContainerWriter extends AbstractFieldWriter implements Comple return mapRoot; } + @Override public void setValueCount(int count) { mapRoot.setValueCount(count); } @@ -89,15 +100,13 @@ public class VectorContainerWriter extends AbstractFieldWriter implements Comple @Override public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) { try { - ValueVector v = mutator.addField(MaterializedField.create(name, type), clazz); + final ValueVector v = mutator.addField(MaterializedField.create(name, type), clazz); putChild(name, v); return this.typeify(v, clazz); } catch (SchemaChangeException e) { throw new IllegalStateException(e); } - } - } @Override
