Repository: arrow Updated Branches: refs/heads/master c45c3b3e1 -> 70c05be21
ARROW-524: provide apis to access nested vectors and buffers Author: Julien Le Dem <jul...@dremio.com> Closes #314 from julienledem/setRangeToOne and squashes the following commits: 0d526bd [Julien Le Dem] ARROW-524: provide apis to access nested vectors and buffers Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/70c05be2 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/70c05be2 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/70c05be2 Branch: refs/heads/master Commit: 70c05be2130bdbb650a83bc46f7c4f8fc8a231df Parents: c45c3b3 Author: Julien Le Dem <jul...@dremio.com> Authored: Sun Feb 5 14:06:37 2017 +0100 Committer: Uwe L. Korn <uw...@xhochy.com> Committed: Sun Feb 5 14:06:37 2017 +0100 ---------------------------------------------------------------------- .../codegen/templates/NullableValueVectors.java | 21 +++-- .../java/org/apache/arrow/vector/BitVector.java | 88 +++++++++++++++++++- .../org/apache/arrow/vector/NullableVector.java | 2 + .../apache/arrow/vector/TestValueVector.java | 36 ++++++++ 4 files changed, 137 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index ce63710..6b25fb3 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -132,6 +132,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type </#if> @Override + public BitVector getValidityVector() { + return bits; + } + + @Override public List<BufferBacked> getFieldInnerVectors() { return innerVectors; } @@ -426,7 +431,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type mutator.fillEmpties(thisIndex); </#if> values.copyFromSafe(fromIndex, thisIndex, from); - bits.getMutator().setSafe(thisIndex, 1); + bits.getMutator().setSafeToOne(thisIndex); } public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){ @@ -525,7 +530,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type @Override public void setIndexDefined(int index){ - bits.getMutator().set(index, 1); + bits.getMutator().setToOne(index); } /** @@ -543,7 +548,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type valuesMutator.set(i, emptyByteArray); } </#if> - bitsMutator.set(index, 1); + bitsMutator.setToOne(index); valuesMutator.set(index, value); <#if type.major == "VarLen">lastSet = index;</#if> } @@ -574,7 +579,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#else> fillEmpties(index); - bits.getMutator().setSafe(index, 1); + bits.getMutator().setSafeToOne(index); values.getMutator().setSafe(index, value, start, length); setCount++; <#if type.major == "VarLen">lastSet = index;</#if> @@ -587,7 +592,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#else> fillEmpties(index); - bits.getMutator().setSafe(index, 1); + bits.getMutator().setSafeToOne(index); values.getMutator().setSafe(index, value, start, length); setCount++; <#if type.major == "VarLen">lastSet = index;</#if> @@ -626,7 +631,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type valuesMutator.set(i, emptyByteArray); } </#if> - bits.getMutator().set(index, 1); + bits.getMutator().setToOne(index); valuesMutator.set(index, holder); <#if type.major == "VarLen">lastSet = index;</#if> } @@ -676,7 +681,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#if type.major == "VarLen"> fillEmpties(index); </#if> - bits.getMutator().setSafe(index, 1); + bits.getMutator().setSafeToOne(index); values.getMutator().setSafe(index, value); setCount++; <#if type.major == "VarLen">lastSet = index;</#if> @@ -687,7 +692,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#if type.major == "VarLen"> fillEmpties(index); </#if> - bits.getMutator().setSafe(index, 1); + bits.getMutator().setSafeToOne(index); values.getMutator().setSafe(index, value); setCount++; } http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index 9beabcb..d1e9abe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -423,8 +423,8 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe * value to set (either 1 or 0) */ public final void set(int index, int value) { - int byteIndex = index >> 3; - int bitIndex = index & 7; + int byteIndex = byteIndex(index); + int bitIndex = bitIndex(index); byte currentByte = data.getByte(byteIndex); byte bitMask = (byte) (1L << bitIndex); if (value != 0) { @@ -432,10 +432,87 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe } else { currentByte -= (bitMask & currentByte); } + data.setByte(byteIndex, currentByte); + } + /** + * Set the bit at the given index to 1. + * + * @param index position of the bit to set + */ + public final void setToOne(int index) { + int byteIndex = byteIndex(index); + int bitIndex = bitIndex(index); + byte currentByte = data.getByte(byteIndex); + byte bitMask = (byte) (1L << bitIndex); + currentByte |= bitMask; data.setByte(byteIndex, currentByte); } + /** + * set count bits to 1 in data starting at firstBitIndex + * @param data the buffer to set + * @param firstBitIndex the index of the first bit to set + * @param count the number of bits to set + */ + public void setRangeToOne(int firstBitIndex, int count) { + int starByteIndex = byteIndex(firstBitIndex); + final int lastBitIndex = firstBitIndex + count; + final int endByteIndex = byteIndex(lastBitIndex); + final int startByteBitIndex = bitIndex(firstBitIndex); + final int endBytebitIndex = bitIndex(lastBitIndex); + if (count < 8 && starByteIndex == endByteIndex) { + // handles the case where we don't have a first and a last byte + byte bitMask = 0; + for (int i = startByteBitIndex; i < endBytebitIndex; ++i) { + bitMask |= (byte) (1L << i); + } + byte currentByte = data.getByte(starByteIndex); + currentByte |= bitMask; + data.setByte(starByteIndex, currentByte); + } else { + // fill in first byte (if it's not full) + if (startByteBitIndex != 0) { + byte currentByte = data.getByte(starByteIndex); + final byte bitMask = (byte) (0xFFL << startByteBitIndex); + currentByte |= bitMask; + data.setByte(starByteIndex, currentByte); + ++ starByteIndex; + } + + // fill in one full byte at a time + for (int i = starByteIndex; i < endByteIndex; i++) { + data.setByte(i, 0xFF); + } + + // fill in the last byte (if it's not full) + if (endBytebitIndex != 0) { + final int byteIndex = byteIndex(lastBitIndex - endBytebitIndex); + byte currentByte = data.getByte(byteIndex); + final byte bitMask = (byte) (0xFFL >>> ((8 - endBytebitIndex) & 7)); + currentByte |= bitMask; + data.setByte(byteIndex, currentByte); + } + + } + } + + /** + * @param absoluteBitIndex the index of the bit in the buffer + * @return the index of the byte containing that bit + */ + private int byteIndex(int absoluteBitIndex) { + return absoluteBitIndex >> 3; + } + + /** + * @param absoluteBitIndex the index of the bit in the buffer + * @return the index of the bit inside the byte + */ + private int bitIndex(int absoluteBitIndex) { + return absoluteBitIndex & 7; + } + public final void set(int index, BitHolder holder) { set(index, holder.value); } @@ -451,6 +528,13 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe set(index, value); } + public void setSafeToOne(int index) { + while(index >= getValueCapacity()) { + reAlloc(); + } + setToOne(index); + } + public void setSafe(int index, BitHolder holder) { while(index >= getValueCapacity()) { reAlloc(); http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java index 0212b3c..b49e916 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java @@ -19,5 +19,7 @@ package org.apache.arrow.vector; public interface NullableVector extends ValueVector { + BitVector getValidityVector(); + ValueVector getValuesVector(); } http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index b33919b..774b59e 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -30,6 +30,7 @@ import org.apache.arrow.vector.schema.TypeLayout; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Field; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -365,6 +366,41 @@ public class TestValueVector { } @Test + public void testBitVectorRangeSetAllOnes() { + validateRange(1000, 0, 1000); + validateRange(1000, 0, 1); + validateRange(1000, 1, 2); + validateRange(1000, 5, 6); + validateRange(1000, 5, 10); + validateRange(1000, 5, 150); + validateRange(1000, 5, 27); + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + validateRange(1000, 10 + i, 27 + j); + validateRange(1000, i, j); + } + } + } + + private void validateRange(int length, int start, int count) { + String desc = "[" + start + ", " + (start + count) + ") "; + try (BitVector bitVector = new BitVector("bits", allocator)) { + bitVector.reset(); + bitVector.allocateNew(length); + bitVector.getMutator().setRangeToOne(start, count); + for (int i = 0; i < start; i++) { + Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i)); + } + for (int i = start; i < start + count; i++) { + Assert.assertEquals(desc + i, 1, bitVector.getAccessor().get(i)); + } + for (int i = start + count; i < length; i++) { + Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i)); + } + } + } + + @Test public void testReAllocNullableFixedWidthVector() { // Create a new value vector for 1024 integers try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {