http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java b/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java deleted file mode 100644 index 84961b1..0000000 --- a/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.vector; - -import static org.junit.Assert.*; - -import org.apache.drill.categories.VectorTest; -import org.apache.drill.common.types.TypeProtos.DataMode; -import org.apache.drill.common.types.TypeProtos.MajorType; -import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.vector.IntVector; -import org.apache.drill.exec.vector.NullableIntVector; -import org.apache.drill.exec.vector.NullableVarCharVector; -import org.apache.drill.exec.vector.RepeatedIntVector; -import org.apache.drill.exec.vector.ValueVector; -import org.apache.drill.exec.vector.VarCharVector; -import org.apache.drill.exec.vector.VectorOverflowException; -import org.apache.drill.test.DrillTest; -import org.apache.drill.test.OperatorFixture; -import org.bouncycastle.util.Arrays; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import io.netty.buffer.DrillBuf; -import org.junit.experimental.categories.Category; - -/** - * Test the setScalar() methods in the various generated vector - * classes. Rather than test all 100+ vectors, we sample a few and - * rely on the fact that code is generated from a common template. - */ - -@Category(VectorTest.class) -public class TestVectorLimits extends DrillTest { - - public static OperatorFixture fixture; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - fixture = OperatorFixture.builder().build(); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - fixture.close(); - } - - /** - * Test a vector directly using the vector mutator to ensure - * that the <tt>setScalar</tt> method works for the maximum - * row count. - * <p> - * This test is a proxy for all the other fixed types, since all - * share the same code template. - */ - - @Test - public void testFixedVector() { - - // Create a non-nullable int vector: a typical fixed-size vector - - @SuppressWarnings("resource") - IntVector vector = new IntVector(makeField(MinorType.INT, DataMode.REQUIRED), fixture.allocator() ); - - // Sanity test of generated constants. - - assertTrue( IntVector.MAX_SCALAR_COUNT <= ValueVector.MAX_ROW_COUNT ); - assertEquals( 4, IntVector.VALUE_WIDTH ); - assertTrue( IntVector.NET_MAX_SCALAR_SIZE <= ValueVector.MAX_BUFFER_SIZE ); - - // Allocate a default size, small vector. Forces test of - // the auto-grow (setSafe()) aspect of setScalar(). - - vector.allocateNew( ); - - // Write to the vector until it complains. At that point, - // we should have written up to the static fixed value count - // (which is computed to stay below the capacity limit.) - - IntVector.Mutator mutator = vector.getMutator(); - for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) { - try { - mutator.setScalar(i, i); - } catch (VectorOverflowException e) { - assertEquals(IntVector.MAX_SCALAR_COUNT, i); - break; - } - } - - // The vector should be below the allocation limit. Since this - // is an int vector, in practice the size will be far below - // the overall limit (if the limit stays at 16 MB.) But, it should - // be at the type-specific limit since we filled up the vector. - - assertEquals(IntVector.NET_MAX_SCALAR_SIZE, vector.getBuffer().getActualMemoryConsumed()); - vector.close(); - } - - @Test - public void testNullableFixedVector() { - - @SuppressWarnings("resource") - NullableIntVector vector = new NullableIntVector(makeField(MinorType.INT, DataMode.OPTIONAL), fixture.allocator() ); - vector.allocateNew( ); - - NullableIntVector.Mutator mutator = vector.getMutator(); - for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) { - try { - mutator.setScalar(i, i); - } catch (VectorOverflowException e) { - assertEquals(IntVector.MAX_SCALAR_COUNT, i); - break; - } - } - - vector.close(); - } - - /** - * Repeated fixed vector. Using an int vector, each column array can hold - * 256 / 4 = 64 values. We write only 10. The vector becomes full when we - * exceed 64K items. - */ - - @Test - public void testRepeatedFixedVectorCountLimit() { - - @SuppressWarnings("resource") - RepeatedIntVector vector = new RepeatedIntVector(makeField(MinorType.INT, DataMode.REPEATED), fixture.allocator() ); - vector.allocateNew( ); - - RepeatedIntVector.Mutator mutator = vector.getMutator(); - top: - for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) { - if (! mutator.startNewValueBounded(i)) { - assertEquals(ValueVector.MAX_ROW_COUNT, i); - // Continue, let's check the addBounded method also - } - for (int j = 0; j < 10; j++) { - try { - mutator.addEntry(i, i * 100 + j); - } catch (VectorOverflowException e) { - assertEquals(ValueVector.MAX_ROW_COUNT, i); - mutator.setValueCount(i); - break top; - } - } - } - - vector.close(); - } - - /** - * Repeated fixed vector. Using an int vector, each column array can hold - * 256 / 4 = 64 values. We write 100. The vector becomes full when we - * exceed the 16 MB size limit. - */ - - @Test - public void testRepeatedFixedVectorBufferLimit() { - - @SuppressWarnings("resource") - RepeatedIntVector vector = new RepeatedIntVector(makeField(MinorType.INT, DataMode.REPEATED), fixture.allocator() ); - vector.allocateNew( ); - - RepeatedIntVector.Mutator mutator = vector.getMutator(); - top: - for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) { - // We'll never hit the value count limit - assertTrue(mutator.startNewValueBounded(i)); - for (int j = 0; j < 100; j++) { - try { - mutator.addEntry(i, i * 100 + j); - } catch (VectorOverflowException e) { - // We should have hit the buffer limit before the value limit. - assertTrue(i < ValueVector.MAX_ROW_COUNT); - mutator.setValueCount(i); - break top; - } - } - } - - vector.close(); - } - - // To be replaced by a test method in a separate commit. - - public static MaterializedField makeField(MinorType dataType, DataMode mode) { - MajorType type = MajorType.newBuilder() - .setMinorType(dataType) - .setMode(mode) - .build(); - - return MaterializedField.create("foo", type); - } - - /** - * Baseline test for a variable-width vector using <tt>setSafe</tt> and - * loading the vector up to the maximum size. Doing so will cause the vector - * to have a buffer that exceeds the maximum size, demonstrating the - * need for <tt>setScalar()</tt>. - */ - - @Test - public void variableVectorBaseline() { - - // Create a non-nullable VarChar vector: a typical variable-size vector - - @SuppressWarnings("resource") - VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() ); - vector.allocateNew( ); - - // A 16 MB value can hold 64K values of up to 256 bytes each. - // To force a size overflow, write values much larger. - // Write the maximum number of values which will silently - // allow the vector to grow beyond the critical size of 16 MB. - // Doing this in production would lead to memory fragmentation. - // So, this is what the setScalar() method assures we don't do. - - byte dummyValue[] = new byte[512]; - Arrays.fill(dummyValue, (byte) 'X'); - VarCharVector.Mutator mutator = vector.getMutator(); - for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) { - mutator.setSafe(i, dummyValue, 0, dummyValue.length); - } - - // The vector should be above the allocation limit. - // This is why code must migrate to the setScalar() call - // away from the setSafe() call. - - assertTrue(ValueVector.MAX_BUFFER_SIZE < vector.getBuffer().getActualMemoryConsumed()); - vector.close(); - } - - /** - * Test a vector directly using the vector mutator to ensure - * that the <tt>setScalar</tt> method works for the maximum - * vector size. - */ - - @Test - public void testWideVariableVector() { - - @SuppressWarnings("resource") - VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() ); - vector.allocateNew( ); - - // A 16 MB value can hold 64K values of up to 256 bytes each. - // To force a size overflow, write values much larger. - // Write to the vector until it complains. At that point, - // we should have written up to the maximum buffer size. - - byte dummyValue[] = makeVarCharValue(512); - VarCharVector.Mutator mutator = vector.getMutator(); - int count = 0; - for ( ; count < 2 * ValueVector.MAX_ROW_COUNT; count++) { - try { - mutator.setScalar(count, dummyValue, 0, dummyValue.length); - } catch (VectorOverflowException e) { - break; - } - } - - // The vector should be at the allocation limit. If it wasn't, we - // should have grown it to hold more data. The value count will - // be below the maximum. - - mutator.setValueCount(count); - assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getBuffer().getActualMemoryConsumed()); - assertTrue(count < ValueVector.MAX_ROW_COUNT); - vector.close(); - } - - private byte[] makeVarCharValue(int n) { - byte dummyValue[] = new byte[n]; - Arrays.fill(dummyValue, (byte) 'X'); - return dummyValue; - } - - @Test - public void testNullableWideVariableVector() { - - @SuppressWarnings("resource") - NullableVarCharVector vector = new NullableVarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() ); - vector.allocateNew( ); - - byte dummyValue[] = makeVarCharValue(512); - NullableVarCharVector.Mutator mutator = vector.getMutator(); - int count = 0; - for ( ; count < 2 * ValueVector.MAX_ROW_COUNT; count++) { - try { - mutator.setScalar(count, dummyValue, 0, dummyValue.length); - } catch (VectorOverflowException e) { - break; - } - } - - mutator.setValueCount(count); - assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getValuesVector().getBuffer().getActualMemoryConsumed()); - assertTrue(count < ValueVector.MAX_ROW_COUNT); - vector.close(); - } - - /** - * Test a vector directly using the vector mutator to ensure - * that the <tt>setScalar</tt> method works for the maximum - * value count. - */ - - @Test - public void testNarrowVariableVector() { - - @SuppressWarnings("resource") - VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() ); - vector.allocateNew( ); - - // Write small values that fit into 16 MB. We should stop writing - // when we reach the value count limit. - - byte dummyValue[] = makeVarCharValue(254); - VarCharVector.Mutator mutator = vector.getMutator(); - int count = 0; - for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) { - try { - mutator.setScalar(count, dummyValue, 0, dummyValue.length); - } catch (VectorOverflowException e) { - break; - } - } - - // Buffer size should be at or below the maximum, with count - // at the maximum. - - mutator.setValueCount(count); - assertTrue(vector.getBuffer().getActualMemoryConsumed() <= ValueVector.MAX_BUFFER_SIZE); - assertEquals(ValueVector.MAX_ROW_COUNT, count); - vector.close(); - } - - /** - * Test a vector directly using the vector mutator to ensure - * that the <tt>setScalar</tt> method works for the maximum - * value count. Uses a DrillBuf as input. - */ - - @Test - public void testDirectVariableVector() { - - @SuppressWarnings("resource") - VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() ); - vector.allocateNew( ); - - // Repeat the big-value test, but with data coming from a DrillBuf - // (direct memory) rather than a heap array. - - @SuppressWarnings("resource") - DrillBuf drillBuf = makeVarCharValueDirect(260); - VarCharVector.Mutator mutator = vector.getMutator(); - int count = 0; - for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) { - try { - mutator.setScalar(count, drillBuf, 0, 260); - } catch (VectorOverflowException e) { - break; - } - } - drillBuf.close(); - - // Again, vector should be at the size limit, count below the - // value limit. - - mutator.setValueCount(count); - assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getBuffer().getActualMemoryConsumed()); - assertTrue(count < ValueVector.MAX_ROW_COUNT); - vector.close(); - } - - private DrillBuf makeVarCharValueDirect(int n) { - byte dummyValue[] = makeVarCharValue(n); - DrillBuf drillBuf = fixture.allocator().buffer(dummyValue.length); - drillBuf.setBytes(0, dummyValue); - return drillBuf; - } - - @Test - public void testDirectNullableVariableVector() { - - @SuppressWarnings("resource") - NullableVarCharVector vector = new NullableVarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() ); - vector.allocateNew( ); - - @SuppressWarnings("resource") - DrillBuf drillBuf = makeVarCharValueDirect(260); - NullableVarCharVector.Mutator mutator = vector.getMutator(); - int count = 0; - for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) { - try { - mutator.setScalar(count, drillBuf, 0, 260); - } catch (VectorOverflowException e) { - break; - } - } - drillBuf.close(); - - mutator.setValueCount(count); - assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getValuesVector().getBuffer().getActualMemoryConsumed()); - assertTrue(count < ValueVector.MAX_ROW_COUNT); - vector.close(); - } - - public static void main(String args[]) { - try { - setUpBeforeClass(); - new TestVectorLimits().performanceTest(); - tearDownAfterClass(); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - private void performanceTest() { - @SuppressWarnings("resource") - VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() ); - byte value[] = makeVarCharValue(1); - int warmCount = 100; - timeSetSafe(vector, value, warmCount); - runSetBounded(vector, value, warmCount); - int runCount = 1000; - timeSetSafe(vector, value, runCount); - runSetBounded(vector, value, runCount); - timeSetSafe(vector, value, runCount); - vector.close(); - } - - private void timeSetSafe(VarCharVector vector, byte[] value, int iterCount) { - long start = System.currentTimeMillis(); - for (int i = 0; i < iterCount; i++) { - vector.clear(); - vector.allocateNew( ); - - VarCharVector.Mutator mutator = vector.getMutator(); - for (int j = 0; j < ValueVector.MAX_ROW_COUNT; j++) { - mutator.setSafe(j, value, 0, value.length); - } - } - long elapsed = System.currentTimeMillis() - start; - System.out.println( iterCount + " runs of setSafe: " + elapsed + " ms." ); - } - - private void runSetBounded(VarCharVector vector, byte[] value, int iterCount) { - long start = System.currentTimeMillis(); - for (int i = 0; i < iterCount; i++) { - vector.clear(); - vector.allocateNew( ); - - VarCharVector.Mutator mutator = vector.getMutator(); - int posn = 0; - for (;;) { - try { - mutator.setScalar(posn++, value, 0, value.length); - } catch (VectorOverflowException e) { - break; - } - } - } - long elapsed = System.currentTimeMillis() - start; - System.out.println( iterCount + " runs of setScalar: " + elapsed + " ms." ); - } -}
http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/jdbc-all/pom.xml ---------------------------------------------------------------------- diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml index 82910af..9db9c58 100644 --- a/exec/jdbc-all/pom.xml +++ b/exec/jdbc-all/pom.xml @@ -509,7 +509,7 @@ This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users. </message> - <maxsize>30000000</maxsize> + <maxsize>32000000</maxsize> <minsize>15000000</minsize> <files> <file>${project.build.directory}/drill-jdbc-all-${project.version}.jar</file> http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java ---------------------------------------------------------------------- diff --git a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java index 9019507..eda189e 100644 --- a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java +++ b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java @@ -52,7 +52,6 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable { private final int offset; private final BufferLedger ledger; private final BufferManager bufManager; -// private final ByteBufAllocator alloc; private final boolean isEmpty; private volatile int length; private final HistoricalLog historicalLog = BaseAllocator.DEBUG ? @@ -72,7 +71,6 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable { this.udle = byteBuf; this.isEmpty = isEmpty; this.bufManager = manager; -// this.alloc = alloc; this.addr = byteBuf.memoryAddress() + offset; this.ledger = ledger; this.length = length; @@ -106,6 +104,8 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable { } } + public long addr() { return addr; } + private long addr(int index) { return addr + index; } @@ -882,4 +882,70 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable { } } + // The "unsafe" methods are for use ONLY by code that does its own + // bounds checking. They are called "unsafe" for a reason: they will crash + // the JVM if values are addressed out of bounds. + + /** + * Write an integer to the buffer at the given byte index, without + * bounds checks. + * + * @param offset byte (not int) offset of the location to write + * @param value the value to write + */ + + public void unsafePutInt(int offset, int value) { + PlatformDependent.putInt(addr + offset, value); + } + + /** + * Write a long to the buffer at the given byte index, without + * bounds checks. + * + * @param index byte (not long) offset of the location to write + * @param value the value to write + */ + + public void unsafePutLong(int index, long value) { + PlatformDependent.putLong(addr + index, value); + } + + /** + * Write a short to the buffer at the given byte index, without + * bounds checks. + * + * @param offset byte (not short) offset of the location to write + * @param value the value to write + */ + + public void unsafePutShort(int offset, short value) { + PlatformDependent.putShort(addr + offset, value); + } + + /** + * Write a byte to the buffer at the given byte index, without + * bounds checks. + * + * @param offset byte offset of the location to write + * @param value the value to write + */ + + public void unsafePutByte(int offset, byte value) { + PlatformDependent.putByte(addr + offset, value); + } + + /** + * Copy a buffer of heap data to the buffer memory. + * + * @param srce source byte buffer + * @param srcOffset offset within the byte buffer of the start of data + * @param destOffset byte offset into this buffer to which to write the + * data + * @param length length of the data, which must be within the + * bounds of this buffer + */ + + public void unsafeCopyMemory(byte[] srce, int srcOffset, int destOffset, int length) { + PlatformDependent.copyMemory(srce, srcOffset, addr + destOffset, length); + } } http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java ---------------------------------------------------------------------- diff --git a/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java index 5358ca2..1e70216 100644 --- a/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java +++ b/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -32,14 +32,16 @@ import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; /** - * The base allocator that we use for all of Drill's memory management. Returns UnsafeDirectLittleEndian buffers. + * The base allocator that we use for all of Drill's memory management. Returns + * UnsafeDirectLittleEndian buffers. */ + public class PooledByteBufAllocatorL { - private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator"); + private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory + .getLogger("drill.allocator"); private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60; - public static final String METRIC_PREFIX = "drill.allocator."; private final MetricRegistry registry; @@ -54,7 +56,8 @@ public class PooledByteBufAllocatorL { public PooledByteBufAllocatorL(MetricRegistry registry) { this.registry = registry; allocator = new InnerAllocator(); - empty = new UnsafeDirectLittleEndian(new DuplicatedByteBuf(Unpooled.EMPTY_BUFFER)); + empty = new UnsafeDirectLittleEndian( + new DuplicatedByteBuf(Unpooled.EMPTY_BUFFER)); } public UnsafeDirectLittleEndian allocate(int size) { @@ -63,7 +66,6 @@ public class PooledByteBufAllocatorL { } catch (OutOfMemoryError e) { throw new OutOfMemoryException("Failure allocating buffer.", e); } - } public int getChunkSize() { @@ -72,13 +74,13 @@ public class PooledByteBufAllocatorL { private class InnerAllocator extends PooledByteBufAllocator { - private final PoolArena<ByteBuffer>[] directArenas; private final MemoryStatusThread statusThread; private final Histogram largeBuffersHist; private final Histogram normalBuffersHist; private final int chunkSize; + @SuppressWarnings("unchecked") public InnerAllocator() { super(true); @@ -87,7 +89,9 @@ public class PooledByteBufAllocatorL { f.setAccessible(true); this.directArenas = (PoolArena<ByteBuffer>[]) f.get(this); } catch (Exception e) { - throw new RuntimeException("Failure while initializing allocator. Unable to retrieve direct arenas field.", e); + throw new RuntimeException( + "Failure while initializing allocator. Unable to retrieve direct arenas field.", + e); } this.chunkSize = directArenas[0].chunkSize; @@ -130,17 +134,14 @@ public class PooledByteBufAllocatorL { largeBuffersHist = registry.histogram(METRIC_PREFIX + "huge.hist"); normalBuffersHist = registry.histogram(METRIC_PREFIX + "normal.hist"); - } - private synchronized void removeOldMetrics() { registry.removeMatching(new MetricFilter() { @Override public boolean matches(String name, Metric metric) { return name.startsWith("drill.allocator."); } - }); } @@ -152,17 +153,21 @@ public class PooledByteBufAllocatorL { if (initialCapacity > directArena.chunkSize) { // This is beyond chunk size so we'll allocate separately. - ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity, maxCapacity); + ByteBuf buf = UnpooledByteBufAllocator.DEFAULT + .directBuffer(initialCapacity, maxCapacity); hugeBufferCount.incrementAndGet(); hugeBufferSize.addAndGet(buf.capacity()); largeBuffersHist.update(buf.capacity()); - // logger.debug("Allocating huge buffer of size {}", initialCapacity, new Exception()); - return new UnsafeDirectLittleEndian(new LargeBuffer(buf, hugeBufferSize, hugeBufferCount)); + // logger.debug("Allocating huge buffer of size {}", initialCapacity, + // new Exception()); + return new UnsafeDirectLittleEndian( + new LargeBuffer(buf, hugeBufferSize, hugeBufferCount)); } else { // within chunk, use arena. - ByteBuf buf = directArena.allocate(cache, initialCapacity, maxCapacity); + ByteBuf buf = directArena.allocate(cache, initialCapacity, + maxCapacity); if (!(buf instanceof PooledUnsafeDirectByteBuf)) { fail(); } @@ -173,10 +178,9 @@ public class PooledByteBufAllocatorL { normalBufferCount.incrementAndGet(); } - return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, normalBufferCount, - normalBufferSize); + return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, + normalBufferCount, normalBufferSize); } - } else { throw fail(); } @@ -184,10 +188,12 @@ public class PooledByteBufAllocatorL { private UnsupportedOperationException fail() { return new UnsupportedOperationException( - "Drill requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality."); + "Drill requires that the JVM used supports access sun.misc.Unsafe. This platform doesn't provide that functionality."); } - public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) { + @Override + public UnsafeDirectLittleEndian directBuffer(int initialCapacity, + int maxCapacity) { if (initialCapacity == 0 && maxCapacity == 0) { newDirectBuffer(initialCapacity, maxCapacity); } @@ -197,13 +203,14 @@ public class PooledByteBufAllocatorL { @Override public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) { - throw new UnsupportedOperationException("Drill doesn't support using heap buffers."); + throw new UnsupportedOperationException( + "Drill doesn't support using heap buffers."); } - private void validate(int initialCapacity, int maxCapacity) { if (initialCapacity < 0) { - throw new IllegalArgumentException("initialCapacity: " + initialCapacity + " (expectd: 0+)"); + throw new IllegalArgumentException( + "initialCapacity: " + initialCapacity + " (expected: 0+)"); } if (initialCapacity > maxCapacity) { throw new IllegalArgumentException(String.format( @@ -223,18 +230,18 @@ public class PooledByteBufAllocatorL { @Override public void run() { while (true) { - memoryLogger.trace("Memory Usage: \n{}", PooledByteBufAllocatorL.this.toString()); + memoryLogger.trace("Memory Usage: \n{}", + PooledByteBufAllocatorL.this.toString()); try { Thread.sleep(MEMORY_LOGGER_FREQUENCY_SECONDS * 1000); } catch (InterruptedException e) { return; } - } } - } + @Override public String toString() { StringBuilder buf = new StringBuilder(); buf.append(directArenas.length); @@ -257,8 +264,6 @@ public class PooledByteBufAllocatorL { buf.append(" bytes."); return buf.toString(); } - - } public static final boolean ASSERT_ENABLED; @@ -268,5 +273,4 @@ public class PooledByteBufAllocatorL { assert isAssertEnabled = true; ASSERT_ENABLED = isAssertEnabled; } - } http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/memory/base/src/main/java/org/apache/drill/exec/memory/AllocationManager.java ---------------------------------------------------------------------- diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/AllocationManager.java b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/AllocationManager.java index 833a604..3b5967f 100644 --- a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/AllocationManager.java +++ b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/AllocationManager.java @@ -38,23 +38,25 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; /** - * Manages the relationship between one or more allocators and a particular UDLE. Ensures that one allocator owns the - * memory that multiple allocators may be referencing. Manages a BufferLedger between each of its associated allocators. - * This class is also responsible for managing when memory is allocated and returned to the Netty-based - * PooledByteBufAllocatorL. + * Manages the relationship between one or more allocators and a particular + * UDLE. Ensures that one allocator owns the memory that multiple allocators may + * be referencing. Manages a BufferLedger between each of its associated + * allocators. This class is also responsible for managing when memory is + * allocated and returned to the Netty-based PooledByteBufAllocatorL. * - * The only reason that this isn't package private is we're forced to put DrillBuf in Netty's package which need access - * to these objects or methods. + * The only reason that this isn't package private is we're forced to put + * DrillBuf in Netty's package which need access to these objects or methods. * - * Threading: AllocationManager manages thread-safety internally. Operations within the context of a single BufferLedger - * are lockless in nature and can be leveraged by multiple threads. Operations that cross the context of two ledgers - * will acquire a lock on the AllocationManager instance. Important note, there is one AllocationManager per - * UnsafeDirectLittleEndian buffer allocation. As such, there will be thousands of these in a typical query. The + * Threading: AllocationManager manages thread-safety internally. Operations + * within the context of a single BufferLedger are lockless in nature and can be + * leveraged by multiple threads. Operations that cross the context of two + * ledgers will acquire a lock on the AllocationManager instance. Important + * note, there is one AllocationManager per UnsafeDirectLittleEndian buffer + * allocation. As such, there will be thousands of these in a typical query. The * contention of acquiring a lock on AllocationManager should be very low. - * */ + public class AllocationManager { - // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationManager.class); private static final AtomicLong MANAGER_ID_GENERATOR = new AtomicLong(0); private static final AtomicLong LEDGER_ID_GENERATOR = new AtomicLong(0); @@ -87,11 +89,13 @@ public class AllocationManager { } /** - * Associate the existing underlying buffer with a new allocator. This will increase the reference count to the - * provided ledger by 1. + * Associate the existing underlying buffer with a new allocator. This will + * increase the reference count to the provided ledger by 1. + * * @param allocator * The target allocator to associate this buffer with. - * @return The Ledger (new or existing) that associates the underlying buffer to this new ledger. + * @return The Ledger (new or existing) that associates the underlying buffer + * to this new ledger. */ BufferLedger associate(final BaseAllocator allocator) { return associate(allocator, true); @@ -114,7 +118,6 @@ public class AllocationManager { } return ledger; } - } try (AutoCloseableLock write = writeLock.open()) { // we have to recheck existing ledger since a second reader => writer could be competing with us. @@ -138,11 +141,16 @@ public class AllocationManager { } } + public static int chunkSize() { + return INNER_ALLOCATOR.getChunkSize(); + } /** - * The way that a particular BufferLedger communicates back to the AllocationManager that it now longer needs to hold - * a reference to particular piece of memory. + * The way that a particular BufferLedger communicates back to the + * AllocationManager that it now longer needs to hold a reference to + * particular piece of memory. */ + private class ReleaseListener { private final BufferAllocator allocator; @@ -154,6 +162,7 @@ public class AllocationManager { /** * Can only be called when you already hold the writeLock. */ + public void release() { allocator.assertOpen(); @@ -180,16 +189,17 @@ public class AllocationManager { throw new IllegalStateException("The final removal of a ledger should be connected to the owning ledger."); } } - - } } /** - * The reference manager that binds an allocator manager to a particular BaseAllocator. Also responsible for creating - * a set of DrillBufs that share a common fate and set of reference counts. - * As with AllocationManager, the only reason this is public is due to DrillBuf being in io.netty.buffer package. + * The reference manager that binds an allocator manager to a particular + * BaseAllocator. Also responsible for creating a set of DrillBufs that share + * a common fate and set of reference counts. As with AllocationManager, the + * only reason this is public is due to DrillBuf being in io.netty.buffer + * package. */ + public class BufferLedger { private final IdentityHashMap<DrillBuf, Object> buffers = @@ -294,7 +304,6 @@ public class AllocationManager { } } } - } private void inc() { @@ -302,9 +311,11 @@ public class AllocationManager { } /** - * Decrement the ledger's reference count. If the ledger is decremented to zero, this ledger should release its - * ownership back to the AllocationManager + * Decrement the ledger's reference count. If the ledger is decremented to + * zero, this ledger should release its ownership back to the + * AllocationManager */ + public int decrement(int decrement) { allocator.assertOpen(); @@ -321,29 +332,36 @@ public class AllocationManager { } /** - * Returns the ledger associated with a particular BufferAllocator. If the BufferAllocator doesn't currently have a - * ledger associated with this AllocationManager, a new one is created. This is placed on BufferLedger rather than - * AllocationManager directly because DrillBufs don't have access to AllocationManager and they are the ones - * responsible for exposing the ability to associate multiple allocators with a particular piece of underlying - * memory. Note that this will increment the reference count of this ledger by one to ensure the ledger isn't - * destroyed before use. + * Returns the ledger associated with a particular BufferAllocator. If the + * BufferAllocator doesn't currently have a ledger associated with this + * AllocationManager, a new one is created. This is placed on BufferLedger + * rather than AllocationManager directly because DrillBufs don't have + * access to AllocationManager and they are the ones responsible for + * exposing the ability to associate multiple allocators with a particular + * piece of underlying memory. Note that this will increment the reference + * count of this ledger by one to ensure the ledger isn't destroyed before + * use. * * @param allocator * @return */ + public BufferLedger getLedgerForAllocator(BufferAllocator allocator) { return associate((BaseAllocator) allocator); } /** - * Create a new DrillBuf associated with this AllocationManager and memory. Does not impact reference count. - * Typically used for slicing. + * Create a new DrillBuf associated with this AllocationManager and memory. + * Does not impact reference count. Typically used for slicing. + * * @param offset * The offset in bytes to start this new DrillBuf. * @param length * The length in bytes that this DrillBuf will provide access to. - * @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger + * @return A new DrillBuf that shares references with all DrillBufs + * associated with this BufferLedger */ + public DrillBuf newDrillBuf(int offset, int length) { allocator.assertOpen(); return newDrillBuf(offset, length, null); @@ -394,6 +412,7 @@ public class AllocationManager { * * @return Size in bytes */ + public int getSize() { return size; } http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/vector/src/main/codegen/templates/ColumnAccessors.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/ColumnAccessors.java b/exec/vector/src/main/codegen/templates/ColumnAccessors.java index f1fbf2f..33b12be 100644 --- a/exec/vector/src/main/codegen/templates/ColumnAccessors.java +++ b/exec/vector/src/main/codegen/templates/ColumnAccessors.java @@ -19,145 +19,131 @@ <@pp.dropOutputFile /> <@pp.changeOutputFile name="/org/apache/drill/exec/vector/accessor/ColumnAccessors.java" /> <#include "/@includes/license.ftl" /> -<#macro getType label> +<#macro getType drillType label> @Override public ValueType valueType() { <#if label == "Int"> return ValueType.INTEGER; + <#elseif drillType == "VarChar" || drillType == "Var16Char"> + return ValueType.STRING; <#else> return ValueType.${label?upper_case}; </#if> } </#macro> -<#macro bindReader prefix drillType> +<#macro bindReader vectorPrefix drillType isArray > <#if drillType = "Decimal9" || drillType == "Decimal18"> - private MaterializedField field; + private MajorType type; </#if> - private ${prefix}${drillType}Vector.Accessor accessor; + private ${vectorPrefix}${drillType}Vector.Accessor accessor; @Override - public void bind(RowIndex vectorIndex, ValueVector vector) { - bind(vectorIndex); + public void bindVector(ValueVector vector) { <#if drillType = "Decimal9" || drillType == "Decimal18"> - field = vector.getField(); + type = vector.getField().getType(); </#if> - accessor = ((${prefix}${drillType}Vector) vector).getAccessor(); + accessor = ((${vectorPrefix}${drillType}Vector) vector).getAccessor(); } <#if drillType = "Decimal9" || drillType == "Decimal18"> @Override - public void bind(RowIndex vectorIndex, MaterializedField field, VectorAccessor va) { - bind(vectorIndex, field, va); - this.field = field; + public void bindVector(MajorType type, VectorAccessor va) { + super.bindVector(type, va); + this.type = type; } </#if> - private ${prefix}${drillType}Vector.Accessor accessor() { + private ${vectorPrefix}${drillType}Vector.Accessor accessor() { if (vectorAccessor == null) { return accessor; } else { - return ((${prefix}${drillType}Vector) vectorAccessor.vector()).getAccessor(); + return ((${vectorPrefix}${drillType}Vector) vectorAccessor.vector()).getAccessor(); } } </#macro> <#macro get drillType accessorType label isArray> @Override public ${accessorType} get${label}(<#if isArray>int index</#if>) { + <#assign getObject ="getObject"/> <#if isArray> - <#assign index=", index"/> - <#assign getObject="getSingleObject"> + <#assign indexVar = "index"/> <#else> - <#assign index=""/> - <#assign getObject="getObject"> + <#assign indexVar = ""/> </#if> - <#if drillType == "VarChar"> - return new String(accessor().get(vectorIndex.index()${index}), Charsets.UTF_8); - <#elseif drillType == "Var16Char"> - return new String(accessor().get(vectorIndex.index()${index}), Charsets.UTF_16); - <#elseif drillType == "VarBinary"> - return accessor().get(vectorIndex.index()${index}); + <#if drillType == "VarChar" || drillType == "Var16Char" || drillType == "VarBinary"> + return accessor().get(vectorIndex.vectorIndex(${indexVar})); <#elseif drillType == "Decimal9" || drillType == "Decimal18"> return DecimalUtility.getBigDecimalFromPrimitiveTypes( - accessor().get(vectorIndex.index()${index}), - field.getScale(), - field.getPrecision()); + accessor().get(vectorIndex.vectorIndex(${indexVar})), + type.getScale(), + type.getPrecision()); <#elseif accessorType == "BigDecimal" || accessorType == "Period"> - return accessor().${getObject}(vectorIndex.index()${index}); + return accessor().${getObject}(vectorIndex.vectorIndex(${indexVar})); + <#elseif drillType == "UInt1"> + return ((int) accessor().get(vectorIndex.vectorIndex(${indexVar}))) & 0xFF; <#else> - return accessor().get(vectorIndex.index()${index}); + return accessor().get(vectorIndex.vectorIndex(${indexVar})); </#if> } -</#macro> -<#macro bindWriter prefix drillType> - <#if drillType = "Decimal9" || drillType == "Decimal18"> - private MaterializedField field; - </#if> - private ${prefix}${drillType}Vector.Mutator mutator; + <#if drillType == "VarChar"> @Override - public void bind(RowIndex vectorIndex, ValueVector vector) { - bind(vectorIndex); - <#if drillType = "Decimal9" || drillType == "Decimal18"> - field = vector.getField(); - </#if> - this.mutator = ((${prefix}${drillType}Vector) vector).getMutator(); + public String getString(<#if isArray>int index</#if>) { + return new String(getBytes(${indexVar}), Charsets.UTF_8); } -</#macro> -<#macro set drillType accessorType label nullable verb> - @Override - public void set${label}(${accessorType} value) { - <#if drillType == "VarChar"> - byte bytes[] = value.getBytes(Charsets.UTF_8); - mutator.${verb}Safe(vectorIndex.index(), bytes, 0, bytes.length); <#elseif drillType == "Var16Char"> - byte bytes[] = value.getBytes(Charsets.UTF_16); - mutator.${verb}Safe(vectorIndex.index(), bytes, 0, bytes.length); - <#elseif drillType == "VarBinary"> - mutator.${verb}Safe(vectorIndex.index(), value, 0, value.length); - <#elseif drillType == "Decimal9"> - mutator.${verb}Safe(vectorIndex.index(), - DecimalUtility.getDecimal9FromBigDecimal(value, - field.getScale(), field.getPrecision())); - <#elseif drillType == "Decimal18"> - mutator.${verb}Safe(vectorIndex.index(), - DecimalUtility.getDecimal18FromBigDecimal(value, - field.getScale(), field.getPrecision())); - <#elseif drillType == "IntervalYear"> - mutator.${verb}Safe(vectorIndex.index(), value.getYears() * 12 + value.getMonths()); - <#elseif drillType == "IntervalDay"> - mutator.${verb}Safe(vectorIndex.index(),<#if nullable> 1,</#if> - value.getDays(), - ((value.getHours() * 60 + value.getMinutes()) * 60 + - value.getSeconds()) * 1000 + value.getMillis()); - <#elseif drillType == "Interval"> - mutator.${verb}Safe(vectorIndex.index(),<#if nullable> 1,</#if> - value.getYears() * 12 + value.getMonths(), - value.getDays(), - ((value.getHours() * 60 + value.getMinutes()) * 60 + - value.getSeconds()) * 1000 + value.getMillis()); + + @Override + public String getString(<#if isArray>int index</#if>) { + return new String(getBytes(${indexVar}), Charsets.UTF_16); + } + </#if> +</#macro> +<#macro build types vectorType accessorType> + <#if vectorType == "Repeated"> + <#assign fnPrefix = "Array" /> + <#assign classType = "Element" /> <#else> - mutator.${verb}Safe(vectorIndex.index(), <#if cast=="set">(${javaType}) </#if>value); + <#assign fnPrefix = vectorType /> + <#assign classType = "Scalar" /> </#if> - } + <#if vectorType == "Required"> + <#assign vectorPrefix = "" /> + <#else> + <#assign vectorPrefix = vectorType /> + </#if> + public static void define${fnPrefix}${accessorType}s( + Class<? extends Base${classType}${accessorType}> ${accessorType?lower_case}s[]) { + <#list types as type> + <#list type.minor as minor> + <#assign drillType=minor.class> + <#assign notyet=minor.accessorDisabled!type.accessorDisabled!false> + <#if ! notyet> + <#assign typeEnum=drillType?upper_case> + ${accessorType?lower_case}s[MinorType.${typeEnum}.ordinal()] = ${vectorPrefix}${drillType}Column${accessorType}.class; + </#if> + </#list> + </#list> + } </#macro> package org.apache.drill.exec.vector.accessor; import java.math.BigDecimal; -import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.vector.*; -import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.util.DecimalUtility; -import org.apache.drill.exec.vector.accessor.impl.AbstractColumnReader; -import org.apache.drill.exec.vector.accessor.impl.AbstractColumnWriter; -import org.apache.drill.exec.vector.complex.BaseRepeatedValueVector; -import org.apache.drill.exec.vector.accessor.impl.AbstractArrayReader; -import org.apache.drill.exec.vector.accessor.impl.AbstractArrayWriter; -import org.apache.drill.exec.vector.accessor.impl.AbstractColumnReader.VectorAccessor; +import org.apache.drill.exec.vector.accessor.reader.BaseScalarReader; +import org.apache.drill.exec.vector.accessor.reader.BaseElementReader; +import org.apache.drill.exec.vector.accessor.reader.VectorAccessor; +import org.apache.drill.exec.vector.accessor.writer.BaseScalarWriter; +import org.apache.drill.exec.vector.accessor.writer.AbstractFixedWidthWriter.BaseFixedWidthWriter; +import org.apache.drill.exec.vector.accessor.writer.BaseVarWidthWriter; import com.google.common.base.Charsets; + import org.joda.time.Period; /** @@ -191,141 +177,176 @@ public class ColumnAccessors { <#if accessorType=="BigDecimal"> <#assign label="Decimal"> </#if> + <#if drillType == "VarChar" || drillType == "Var16Char"> + <#assign accessorType = "byte[]"> + <#assign label = "Bytes"> + </#if> <#if ! notyet> //------------------------------------------------------------------------ // ${drillType} readers and writers - public static class ${drillType}ColumnReader extends AbstractColumnReader { + public static class ${drillType}ColumnReader extends BaseScalarReader { - <@bindReader "" drillType /> + <@bindReader "" drillType false /> - <@getType label /> + <@getType drillType label /> <@get drillType accessorType label false/> } - public static class Nullable${drillType}ColumnReader extends AbstractColumnReader { + public static class Nullable${drillType}ColumnReader extends BaseScalarReader { - <@bindReader "Nullable" drillType /> + <@bindReader "Nullable" drillType false /> - <@getType label /> + <@getType drillType label /> @Override public boolean isNull() { - return accessor().isNull(vectorIndex.index()); - } - - <@get drillType accessorType label false/> - } - - public static class Repeated${drillType}ColumnReader extends AbstractArrayReader { - - <@bindReader "Repeated" drillType /> - - <@getType label /> - - @Override - public int size() { - return accessor().getInnerValueCountAt(vectorIndex.index()); + return accessor().isNull(vectorIndex.vectorIndex()); } - <@get drillType accessorType label true/> + <@get drillType accessorType label false /> } - public static class ${drillType}ColumnWriter extends AbstractColumnWriter { + public static class Repeated${drillType}ColumnReader extends BaseElementReader { - <@bindWriter "" drillType /> + <@bindReader "" drillType true /> - <@getType label /> + <@getType drillType label /> - <@set drillType accessorType label false "set" /> + <@get drillType accessorType label true /> } - public static class Nullable${drillType}ColumnWriter extends AbstractColumnWriter { - - <@bindWriter "Nullable" drillType /> + <#assign varWidth = drillType == "VarChar" || drillType == "Var16Char" || drillType == "VarBinary" /> + <#if varWidth> + public static class ${drillType}ColumnWriter extends BaseVarWidthWriter { + <#else> + public static class ${drillType}ColumnWriter extends BaseFixedWidthWriter { + <#if drillType = "Decimal9" || drillType == "Decimal18" || + drillType == "Decimal28Sparse" || drillType == "Decimal38Sparse"> + private MajorType type; + </#if> + private static final int VALUE_WIDTH = ${drillType}Vector.VALUE_WIDTH; + </#if> + private final ${drillType}Vector vector; + + public ${drillType}ColumnWriter(final ValueVector vector) { + <#if varWidth> + super(((${drillType}Vector) vector).getOffsetVector()); + <#else> + <#if drillType = "Decimal9" || drillType == "Decimal18" || + drillType == "Decimal28Sparse" || drillType == "Decimal38Sparse"> + type = vector.getField().getType(); + </#if> + </#if> + this.vector = (${drillType}Vector) vector; + } - <@getType label /> + @Override public BaseDataValueVector vector() { return vector; } + + <#if ! varWidth> + @Override public int width() { return VALUE_WIDTH; } + + </#if> + <@getType drillType label /> + <#if accessorType == "byte[]"> + <#assign args = ", int len"> + <#else> + <#assign args = ""> + </#if> + <#if javaType == "char"> + <#assign putType = "short" /> + <#assign doCast = true /> + <#else> + <#assign putType = javaType /> + <#assign doCast = (cast == "set") /> + </#if> + <#if ! varWidth> + </#if> @Override - public void setNull() { - mutator.setNull(vectorIndex.index()); + public final void set${label}(final ${accessorType} value${args}) { + <#-- Must compute the write offset first; can't be inline because the + writeOffset() function has a side effect of possibly changing the buffer + address (bufAddr). --> + <#if varWidth> + final int offset = writeIndex(len); + <#else> + final int writeIndex = writeIndex(); + <#assign putAddr = "writeIndex * VALUE_WIDTH"> + </#if> + <#if varWidth> + drillBuf.unsafeCopyMemory(value, 0, offset, len); + offsetsWriter.setNextOffset(offset + len); + <#elseif drillType == "Decimal9"> + drillBuf.unsafePutInt(${putAddr}, + DecimalUtility.getDecimal9FromBigDecimal(value, + type.getScale(), type.getPrecision())); + <#elseif drillType == "Decimal18"> + drillBuf.unsafePutLong(${putAddr}, + DecimalUtility.getDecimal18FromBigDecimal(value, + type.getScale(), type.getPrecision())); + <#elseif drillType == "Decimal38Sparse"> + <#-- Hard to optimize this case. Just use the available tools. --> + DecimalUtility.getSparseFromBigDecimal(value, vector.getBuffer(), writeIndex * VALUE_WIDTH, + type.getScale(), type.getPrecision(), 6); + <#elseif drillType == "Decimal28Sparse"> + <#-- Hard to optimize this case. Just use the available tools. --> + DecimalUtility.getSparseFromBigDecimal(value, vector.getBuffer(), writeIndex * VALUE_WIDTH, + type.getScale(), type.getPrecision(), 5); + <#elseif drillType == "IntervalYear"> + drillBuf.unsafePutInt(${putAddr}, + value.getYears() * 12 + value.getMonths()); + <#elseif drillType == "IntervalDay"> + final int offset = ${putAddr}; + drillBuf.unsafePutInt(offset, value.getDays()); + drillBuf.unsafePutInt(offset + 4, periodToMillis(value)); + <#elseif drillType == "Interval"> + final int offset = ${putAddr}; + drillBuf.unsafePutInt(offset, value.getYears() * 12 + value.getMonths()); + drillBuf.unsafePutInt(offset + 4, value.getDays()); + drillBuf.unsafePutInt(offset + 8, periodToMillis(value)); + <#elseif drillType == "Float4"> + drillBuf.unsafePutInt(${putAddr}, Float.floatToRawIntBits((float) value)); + <#elseif drillType == "Float8"> + drillBuf.unsafePutLong(${putAddr}, Double.doubleToRawLongBits(value)); + <#else> + drillBuf.unsafePut${putType?cap_first}(${putAddr}, <#if doCast>(${putType}) </#if>value); + </#if> + vectorIndex.nextElement(); } - - <@set drillType accessorType label true "set" /> - } - - public static class Repeated${drillType}ColumnWriter extends AbstractArrayWriter { - - <@bindWriter "Repeated" drillType /> - - <@getType label /> - - protected BaseRepeatedValueVector.BaseRepeatedMutator mutator() { - return mutator; + <#if drillType == "VarChar"> + + @Override + public final void setString(String value) { + final byte bytes[] = value.getBytes(Charsets.UTF_8); + setBytes(bytes, bytes.length); + } + <#elseif drillType == "Var16Char"> + + @Override + public final void setString(String value) { + final byte bytes[] = value.getBytes(Charsets.UTF_16); + setBytes(bytes, bytes.length); } - - <@set drillType accessorType label false "add" /> - } - - </#if> - </#list> -</#list> - public static void defineReaders( - Class<? extends AbstractColumnReader> readers[][]) { -<#list vv.types as type> - <#list type.minor as minor> - <#assign drillType=minor.class> - <#assign notyet=minor.accessorDisabled!type.accessorDisabled!false> - <#if ! notyet> - <#assign typeEnum=drillType?upper_case> - readers[MinorType.${typeEnum}.ordinal()][DataMode.REQUIRED.ordinal()] = ${drillType}ColumnReader.class; - readers[MinorType.${typeEnum}.ordinal()][DataMode.OPTIONAL.ordinal()] = Nullable${drillType}ColumnReader.class; </#if> - </#list> -</#list> } - public static void defineWriters( - Class<? extends AbstractColumnWriter> writers[][]) { -<#list vv.types as type> - <#list type.minor as minor> - <#assign drillType=minor.class> - <#assign notyet=minor.accessorDisabled!type.accessorDisabled!false> - <#if ! notyet> - <#assign typeEnum=drillType?upper_case> - writers[MinorType.${typeEnum}.ordinal()][DataMode.REQUIRED.ordinal()] = ${drillType}ColumnWriter.class; - writers[MinorType.${typeEnum}.ordinal()][DataMode.OPTIONAL.ordinal()] = Nullable${drillType}ColumnWriter.class; </#if> </#list> </#list> + public static int periodToMillis(Period value) { + return ((value.getHours() * 60 + + value.getMinutes()) * 60 + + value.getSeconds()) * 1000 + + value.getMillis(); } - public static void defineArrayReaders( - Class<? extends AbstractArrayReader> readers[]) { -<#list vv.types as type> - <#list type.minor as minor> - <#assign drillType=minor.class> - <#assign notyet=minor.accessorDisabled!type.accessorDisabled!false> - <#if ! notyet> - <#assign typeEnum=drillType?upper_case> - readers[MinorType.${typeEnum}.ordinal()] = Repeated${drillType}ColumnReader.class; - </#if> - </#list> -</#list> - } +<@build vv.types "Required" "Reader" /> - public static void defineArrayWriters( - Class<? extends AbstractArrayWriter> writers[]) { -<#list vv.types as type> - <#list type.minor as minor> - <#assign drillType=minor.class> - <#assign notyet=minor.accessorDisabled!type.accessorDisabled!false> - <#if ! notyet> - <#assign typeEnum=drillType?upper_case> - writers[MinorType.${typeEnum}.ordinal()] = Repeated${drillType}ColumnWriter.class; - </#if> - </#list> -</#list> - } +<@build vv.types "Nullable" "Reader" /> + +<@build vv.types "Repeated" "Reader" /> + +<@build vv.types "Required" "Writer" /> } http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/vector/src/main/codegen/templates/FixedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java index 51938a3..1f6a008 100644 --- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java @@ -22,8 +22,8 @@ <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> <#if type.major == "Fixed"> -<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" /> -<#include "/@includes/license.ftl" /> + <@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" /> + <#include "/@includes/license.ftl" /> package org.apache.drill.exec.vector; @@ -31,15 +31,18 @@ package org.apache.drill.exec.vector; import org.apache.drill.exec.util.DecimalUtility; /** - * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed - * by position, starting from the logical start of the vector. Values should be pushed onto the - * vector sequentially, but may be accessed randomly. + * ${minor.class} implements a vector of fixed width values. Elements in the + * vector are accessed by position, starting from the logical start of the + * vector. Values should be pushed onto the vector sequentially, but may be + * accessed randomly. * <ul> - * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width}) byte<#if type.width != 1>s</#if>.</li> + * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width}) + * byte<#if type.width != 1>s</#if>.</li> * <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'.</li> * </ul> * - * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. + * NB: this class is automatically generated from ${.template_name} and + * ValueVectorTypes.tdd using FreeMarker. */ public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); @@ -151,13 +154,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } /** - * Allocate a new buffer that supports setting at least the provided number of values. May actually be sized bigger - * depending on underlying buffer rounding size. Must be called prior to using the ValueVector. + * Allocate a new buffer that supports setting at least the provided number of + * values. May actually be sized bigger depending on underlying buffer + * rounding size. Must be called prior to using the ValueVector. * - * Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width. + * Note that the maximum number of values a vector can allocate is + * Integer.MAX_VALUE / value width. * * @param valueCount - * @throws OutOfMemoryException if it can't allocate the new buffer + * @throws OutOfMemoryException + * if it can't allocate the new buffer */ @Override public void allocateNew(final int valueCount) { @@ -211,18 +217,24 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached."); } + reallocRaw((int) newAllocationSize); + final int halfNewCapacity = data.capacity() / 2; + data.setZero(halfNewCapacity, halfNewCapacity); + } + + @Override + public DrillBuf reallocRaw(int newAllocationSize) { logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize); if (newAllocationSize == 0) { throw new IllegalStateException("Attempt to reAlloc a zero-sized vector"); } - final DrillBuf newBuf = allocator.buffer((int)newAllocationSize); + final DrillBuf newBuf = allocator.buffer(newAllocationSize); newBuf.setBytes(0, data, 0, data.capacity()); - final int halfNewCapacity = newBuf.capacity() / 2; - newBuf.setZero(halfNewCapacity, halfNewCapacity); newBuf.writerIndex(data.writerIndex()); data.release(1); data = newBuf; - allocationSizeInBytes = (int)newAllocationSize; + allocationSizeInBytes = newAllocationSize; + return newBuf; } /** @@ -337,7 +349,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public void copyEntry(int toIndex, ValueVector from, int fromIndex) { - ((${minor.class}Vector) from).data.getBytes(fromIndex * ${type.width}, data, toIndex * ${type.width}, ${type.width}); + ((${minor.class}Vector) from).data.getBytes(fromIndex * VALUE_WIDTH, data, toIndex * VALUE_WIDTH, VALUE_WIDTH); } public void decrementAllocationMonitor() { @@ -423,7 +435,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F final String monthString = (Math.abs(months) == 1) ? " month " : " months "; final String dayString = (Math.abs(days) == 1) ? " day " : " days "; - return(new StringBuilder(). append(years).append(yearString). append(months).append(monthString). @@ -621,26 +632,31 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } /** - * ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the - * vector are accessed by position from the logical start of the vector. Values should be pushed - * onto the vector sequentially, but may be randomly accessed. + * ${minor.class}.Mutator implements a mutable vector of fixed width values. + * Elements in the vector are accessed by position from the logical start of + * the vector. Values should be pushed onto the vector sequentially, but may + * be randomly accessed. * <ul> - * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width}) byte(s).</li> + * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width}) + * byte(s).</li> * <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'</li> * </ul> * - * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker. + * NB: this class is automatically generated from ValueVectorTypes.tdd using + * FreeMarker. */ public final class Mutator extends BaseDataValueVector.BaseMutator { private Mutator() {}; /** - * Set the element at the given index to the given value. Note that widths smaller than - * 32 bits are handled by the DrillBuf interface. + * Set the element at the given index to the given value. Note that widths + * smaller than 32 bits are handled by the DrillBuf interface. * - * @param index position of the bit to set - * @param value value to set + * @param index + * position of the bit to set + * @param value + * value to set */ <#if (type.width > 8)> @@ -655,37 +671,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F data.setBytes(index * VALUE_WIDTH, value, 0, VALUE_WIDTH); } - /** - * Set the value of a required or nullable vector. Enforces the value - * and size limits. - * @param index item to write - * @param value value to set - * @throws VectorOverflowException if the item was written, false if the index would - * overfill the vector - */ - - public void setScalar(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException { - if (index >= MAX_SCALAR_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, value); - } - - /** - * Set the value of a repeated vector. Enforces only the size limit. - * @param index item to write - * @param value value to set - * @throws VectorOverflowException if the item was written, false if the index would - * overfill the vector - */ - - public void setArrayItem(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException { - if (index >= MAX_VALUE_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, value); - } - <#if minor.class == "Interval"> public void set(int index, int months, int days, int milliseconds) { final int offsetIndex = index * VALUE_WIDTH; @@ -701,20 +686,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F set(index, months, days, milliseconds); } - public void setScalar(int index, int months, int days, int milliseconds) throws VectorOverflowException { - if (index >= MAX_SCALAR_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, months, days, milliseconds); - } - - public void setArrayItem(int index, int months, int days, int milliseconds) throws VectorOverflowException { - if (index >= MAX_VALUE_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, months, days, milliseconds); - } - protected void set(int index, ${minor.class}Holder holder) { set(index, holder.months, holder.days, holder.milliseconds); } @@ -723,14 +694,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F setSafe(index, holder.months, holder.days, holder.milliseconds); } - public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException { - setScalar(index, holder.months, holder.days, holder.milliseconds); - } - - public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException { - setArrayItem(index, holder.months, holder.days, holder.milliseconds); - } - protected void set(int index, Nullable${minor.class}Holder holder) { set(index, holder.months, holder.days, holder.milliseconds); } @@ -739,14 +702,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F setSafe(index, holder.months, holder.days, holder.milliseconds); } - public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { - setScalar(index, holder.months, holder.days, holder.milliseconds); - } - - public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { - setArrayItem(index, holder.months, holder.days, holder.milliseconds); - } - <#elseif minor.class == "IntervalDay"> public void set(int index, int days, int milliseconds) { final int offsetIndex = index * VALUE_WIDTH; @@ -761,20 +716,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F set(index, days, milliseconds); } - public void setScalar(int index, int days, int milliseconds) throws VectorOverflowException { - if (index >= MAX_SCALAR_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, days, milliseconds); - } - - public void setArrayItem(int index, int days, int milliseconds) throws VectorOverflowException { - if (index >= MAX_VALUE_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, days, milliseconds); - } - protected void set(int index, ${minor.class}Holder holder) { set(index, holder.days, holder.milliseconds); } @@ -783,14 +724,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F setSafe(index, holder.days, holder.milliseconds); } - public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException { - setScalar(index, holder.days, holder.milliseconds); - } - - public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException { - setArrayItem(index, holder.days, holder.milliseconds); - } - protected void set(int index, Nullable${minor.class}Holder holder) { set(index, holder.days, holder.milliseconds); } @@ -799,14 +732,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F setSafe(index, holder.days, holder.milliseconds); } - public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { - setScalar(index, holder.days, holder.milliseconds); - } - - public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { - setArrayItem(index, holder.days, holder.milliseconds); - } - <#elseif minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense"> public void setSafe(int index, int start, DrillBuf buffer) { while(index >= getValueCapacity()) { @@ -815,20 +740,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F set(index, start, buffer); } - public void setScalar(int index, int start, DrillBuf buffer) throws VectorOverflowException { - if (index >= MAX_SCALAR_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, start, buffer); - } - - public void setArrayItem(int index, int start, DrillBuf buffer) throws VectorOverflowException { - if (index >= MAX_VALUE_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, start, buffer); - } - public void set(int index, ${minor.class}Holder holder) { set(index, holder.start, holder.buffer); } @@ -837,14 +748,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F setSafe(index, holder.start, holder.buffer); } - public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException { - setScalar(index, holder.start, holder.buffer); - } - - public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException { - setArrayItem(index, holder.start, holder.buffer); - } - void set(int index, Nullable${minor.class}Holder holder) { set(index, holder.start, holder.buffer); } @@ -853,14 +756,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F setSafe(index, holder.start, holder.buffer); } - public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { - setScalar(index, holder.start, holder.buffer); - } - - public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { - setArrayItem(index, holder.start, holder.buffer); - } - <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse"> public void set(int index, BigDecimal value) { DecimalUtility.getSparseFromBigDecimal(value, data, index * VALUE_WIDTH, @@ -874,20 +769,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F set(index, value); } - public void setScalar(int index, BigDecimal value) throws VectorOverflowException { - if (index >= MAX_SCALAR_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, value); - } - - public void setArrayItem(int index, BigDecimal value) throws VectorOverflowException { - if (index >= MAX_VALUE_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, value); - } - </#if> public void set(int index, int start, DrillBuf buffer){ data.setBytes(index * VALUE_WIDTH, buffer, start, VALUE_WIDTH); @@ -912,42 +793,18 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, value); } - public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) { - while(index >= getValueCapacity()) { - reAlloc(); - } - set(index, value); - } - /** - * Set the value of a required or nullable vector. Enforces the value - * and size limits. + * Set the value of a required or nullable vector. Grows the vector as needed. + * Does not enforce size limits; scalar fixed-width types can never overflow + * a vector. * @param index item to write - * @param value value to set - * @throws VectorOverflowException if the item was written, false if the index would - * overfill the vector */ - public void setScalar(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException { - if (index >= MAX_SCALAR_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, value); - } - - /** - * Set the value of a repeated vector. Enforces only the size limit. - * @param index item to write - * @param value value to set - * @throws VectorOverflowException if the item was written, false if the index would - * overfill the vector - */ - - public void setArrayItem(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException { - if (index >= MAX_VALUE_COUNT) { - throw new VectorOverflowException(); + public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) { + while(index >= getValueCapacity()) { + reAlloc(); } - setSafe(index, value); + set(index, value); } protected void set(int index, ${minor.class}Holder holder) { @@ -961,20 +818,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F set(index, holder); } - public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException { - if (index >= MAX_SCALAR_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, holder); - } - - public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException { - if (index >= MAX_VALUE_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, holder); - } - protected void set(int index, Nullable${minor.class}Holder holder) { data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, holder.value); } @@ -986,20 +829,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F set(index, holder); } - public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { - if (index >= MAX_SCALAR_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, holder); - } - - public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { - if (index >= MAX_VALUE_COUNT) { - throw new VectorOverflowException(); - } - setSafe(index, holder); - } - @Override public void generateTestData(int size) { setValueCount(size); @@ -1028,30 +857,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } </#if> <#-- type.width --> - /** - * Backfill missing offsets from the given last written position to the - * given current write position. Used by the "new" size-safe column - * writers to allow skipping values. The <tt>set()</tt> and <tt>setSafe()</tt> - * <b>do not</b> fill empties. See DRILL-5529 and DRILL-5530. - * @param lastWrite the position of the last valid write: the offset - * to be copied forward - * @param index the current write position filling occurs up to, - * but not including, this position - * @throws VectorOverflowException if the item was written, false if the index would - * overfill the vector - */ - - public void fillEmptiesBounded(int lastWrite, int index) - throws VectorOverflowException { - <#if type.width <= 8> - for (int i = lastWrite + 1; i <= index; i++) { - setSafe(i, <#if (type.width >= 4)>(${minor.javaType!type.javaType})</#if> 0); - } - <#else> - throw new UnsupportedOperationException("Cannot zero-fill ${minor.class} vectors."); - </#if> - } - @Override public void setValueCount(int valueCount) { final int currentValueCapacity = getValueCapacity(); http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/vector/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java index 4f3eb17..fdb0200 100644 --- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java @@ -47,7 +47,7 @@ package org.apache.drill.exec.vector; * * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -@SuppressWarnings("unused") + public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); @@ -180,6 +180,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type } @Override + public DrillBuf reallocRaw(int newAllocationSize) { + throw new UnsupportedOperationException(); + } + + @Override public void collectLedgers(Set<BufferLedger> ledgers) { bits.collectLedgers(ledgers); values.collectLedgers(ledgers); @@ -304,7 +309,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type } private class TransferImpl implements TransferPair { - Nullable${minor.class}Vector to; + private final Nullable${minor.class}Vector to; public TransferImpl(MaterializedField field, BufferAllocator allocator){ to = new Nullable${minor.class}Vector(field, allocator); @@ -336,12 +341,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type } @Override - public Accessor getAccessor(){ + public Accessor getAccessor() { return accessor; } @Override - public Mutator getMutator(){ + public Mutator getMutator() { return mutator; } @@ -548,16 +553,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type lastSet = index; } - public void setScalar(int index, byte[] value, int start, int length) throws VectorOverflowException { - if (index > lastSet + 1) { - fillEmpties(index); // Filling empties cannot overflow the vector - } - values.getMutator().setScalar(index, value, start, length); - bits.getMutator().setSafe(index, 1); - setCount++; - lastSet = index; - } - public void setSafe(int index, ByteBuffer value, int start, int length) { if (index > lastSet + 1) { fillEmpties(index); @@ -569,17 +564,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type lastSet = index; } - public void setScalar(int index, DrillBuf value, int start, int length) throws VectorOverflowException { - if (index > lastSet + 1) { - fillEmpties(index); // Filling empties cannot overflow the vector - } - - values.getMutator().setScalar(index, value, start, length); - bits.getMutator().setSafe(index, 1); - setCount++; - lastSet = index; - } - </#if> public void setNull(int index) { bits.getMutator().setSafe(index, 0); @@ -593,10 +577,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type values.getMutator().set(index, holder); } - public void setNullBounded(int index) throws VectorOverflowException { - bits.getMutator().setScalar(index, 0); - } - public void set(int index, Nullable${minor.class}Holder holder) { final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> @@ -648,18 +628,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>); setCount++; <#if type.major == "VarLen">lastSet = index;</#if> - } - - public void setScalar(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) throws VectorOverflowException { - <#if type.major == "VarLen"> - if (index > lastSet + 1) { - fillEmpties(index); - } - </#if> - values.getMutator().setScalar(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>); - bits.getMutator().setSafe(index, isSet); - setCount++; - <#if type.major == "VarLen">lastSet = index;</#if> } public void setSafe(int index, Nullable${minor.class}Holder value) { @@ -674,18 +642,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#if type.major == "VarLen">lastSet = index;</#if> } - public void setScalar(int index, Nullable${minor.class}Holder value) throws VectorOverflowException { - <#if type.major == "VarLen"> - if (index > lastSet + 1) { - fillEmpties(index); - } - </#if> - values.getMutator().setScalar(index, value); - bits.getMutator().setSafe(index, value.isSet); - setCount++; - <#if type.major == "VarLen">lastSet = index;</#if> - } - public void setSafe(int index, ${minor.class}Holder value) { <#if type.major == "VarLen"> if (index > lastSet + 1) { @@ -698,18 +654,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#if type.major == "VarLen">lastSet = index;</#if> } - public void setScalar(int index, ${minor.class}Holder value) throws VectorOverflowException { - <#if type.major == "VarLen"> - if (index > lastSet + 1) { - fillEmpties(index); - } - </#if> - values.getMutator().setScalar(index, value); - bits.getMutator().setSafe(index, 1); - setCount++; - <#if type.major == "VarLen">lastSet = index;</#if> - } - <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")> public void setSafe(int index, ${minor.javaType!type.javaType} value) { <#if type.major == "VarLen"> @@ -722,17 +666,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type setCount++; } - public void setScalar(int index, ${minor.javaType!type.javaType} value) throws VectorOverflowException { - <#if type.major == "VarLen"> - if (index > lastSet + 1) { - fillEmpties(index); - } - </#if> - values.getMutator().setScalar(index, value); - bits.getMutator().setSafe(index, 1); - setCount++; - } - </#if> <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse"> public void set(int index, BigDecimal value) { @@ -747,12 +680,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type setCount++; } - public void setScalar(int index, BigDecimal value) throws VectorOverflowException { - values.getMutator().setScalar(index, value); - bits.getMutator().setSafe(index, 1); - setCount++; - } - </#if> @Override public void setValueCount(int valueCount) {
