This is an automated email from the ASF dual-hosted git repository.
Fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-java.git
The following commit(s) were added to refs/heads/master by this push:
new 95250a9bc GH-3475: Fix parquet-vector compatiblity with Java > 17
(#3476)
95250a9bc is described below
commit 95250a9bcbf877f6091df9ddb26bf3337c4c1a3b
Author: Ismaël Mejía <[email protected]>
AuthorDate: Sat Jul 11 08:02:07 2026 +0200
GH-3475: Fix parquet-vector compatiblity with Java > 17 (#3476)
* GH-3475: Fix parquet-vector compatiblity with Java > 17
Replace the ByteBuffer-specific vector loads with local helpers that copy
the required bytes and then call ByteVector.fromArray. This removes the
dependency on JDK-specific ByteVector.fromByteBuffer entry points, which can
fail with NoSuchMethodError on newer runtimes.
Assisted-by: OpenCode:gpt-5.4
* GH-3475: Address review comments on parquet-vector ByteBuffer load helpers
Mirror OpenJDK 17's masked fromByteBuffer fast path: read the full
species.length() window into a backing array, then let
ByteVector.fromArray(..., mask) apply the mask. Matches the JDK
semantics for arbitrary mask shapes (not just contiguous-prefix
masks) and keeps array.length == species.length(), satisfying the
bounds-check precondition that the masked fromArray overload may
make.
Add a heap-buffer fast path: when input.hasArray() is true, load
directly from the backing byte[] via ByteVector.fromArray. This
avoids the per-call ByteBuffer.duplicate() and the intermediate
byte[] allocation/copy that the previous workaround introduced as a
regression versus the original ByteVector.fromByteBuffer code path.
The direct (off-heap) buffer case still falls back to a small
scratch array; addressing it requires ByteVector.fromMemorySegment,
which is only available on JDK 19+ and cannot be called from
--release 17 sources.
* GH-3475: Address second round of review comments
- Restore trailing blank line in .gitignore
- Fix masked fromByteBuffer direct-buffer path: allocate species.length()
array but only copy mask.trueCount() bytes, preserving JDK 17 masked-load
bounds semantics (only masked-on lanes must be in bounds)
- Refactor vector-plugins.yml install step to use SPOTLESS_ARGS pattern
matching the verify step
- Add unit tests for direct and read-only ByteBuffer unpack paths
Assisted-by: GitHub Copilot:claude-opus-4.6
* Fix OOM in TestByteBitPacking512VectorLE on CI
The 512-bit vector tests were allocating arrays of 268 million elements
(~1 GB each) and iterating through the entire value space (up to 524K
chunks for bitWidth=32). On the JDK 21 CI runner — which has AVX-512
support so the tests are not skipped — the heap was exceeded, causing
OutOfMemoryError in the DirectByteBuffer and ReadOnlyByteBuffer tests.
Reduce itemMax from 268435456 to 8192 (still 512+ vector iterations per
chunk) and only test the first and last chunks via a stream filter.
This covers both low-range and high-range boundary values while keeping
memory trivial (~33 KB per array) and runtime under 1 second.
Assisted-by: GitHub Copilot:claude-opus-4.6
* Fix Spotless formatting in TestByteBitPacking512VectorLE
Assisted-by: OpenCode:claude-opus-4.6
---------
Co-authored-by: Fokko Driesprong <[email protected]>
---
.github/workflows/vector-plugins.yml | 20 ++-
.gitignore | 2 +-
.../bitpacking/ByteBitPacking512VectorLE.java | 124 ++++++++++++------
.../bitpacking/TestByteBitPacking512VectorLE.java | 143 ++++++++++++++++-----
4 files changed, 211 insertions(+), 78 deletions(-)
diff --git a/.github/workflows/vector-plugins.yml
b/.github/workflows/vector-plugins.yml
index ea4037ae3..bfa9c477a 100644
--- a/.github/workflows/vector-plugins.yml
+++ b/.github/workflows/vector-plugins.yml
@@ -31,7 +31,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ '17' ]
+ java: [ '17', '21', '25' ]
codes: [ 'uncompressed' ]
name: Build Parquet with JDK ${{ matrix.java }} and ${{ matrix.codes }}
@@ -51,7 +51,14 @@ jobs:
run: |
EXTRA_JAVA_TEST_ARGS=$(./mvnw help:evaluate
-Dexpression=extraJavaTestArgs -q -DforceStdout)
export MAVEN_OPTS="$MAVEN_OPTS $EXTRA_JAVA_TEST_ARGS"
- ./mvnw install --batch-mode -Pvector-plugins -DskipTests=true
-Dmaven.javadoc.skip=true -Dsource.skip=true -Dmaven.buildNumber.skip=true
-Djava.version=${{ matrix.java }} -pl
parquet-plugins/parquet-encoding-vector,parquet-plugins/parquet-plugins-benchmarks
-am
+ # Spotless check uses palantir-java-format which relies on internal
javac APIs
+ # that are not available on all JDK versions (e.g. JDK 25+). Since
the formatting
+ # result is JDK-independent, running the check on JDK 17 alone is
sufficient.
+ SPOTLESS_ARGS=""
+ if [ "${{ matrix.java }}" != "17" ]; then
+ SPOTLESS_ARGS="-Dspotless.check.skip=true"
+ fi
+ ./mvnw install --batch-mode -Pvector-plugins -DskipTests=true
-Dmaven.javadoc.skip=true -Dsource.skip=true -Dmaven.buildNumber.skip=true
$SPOTLESS_ARGS -Djava.version=${{ matrix.java }} -pl
parquet-plugins/parquet-encoding-vector,parquet-plugins/parquet-plugins-benchmarks
-am
- name: verify
env:
TEST_CODECS: ${{ matrix.codes }}
@@ -59,4 +66,11 @@ jobs:
run: |
EXTRA_JAVA_TEST_ARGS=$(./mvnw help:evaluate
-Dexpression=extraJavaTestArgs -q -DforceStdout)
export MAVEN_OPTS="$MAVEN_OPTS $EXTRA_JAVA_TEST_ARGS"
- ./mvnw verify --batch-mode -Pvector-plugins javadoc:javadoc -pl
parquet-plugins/parquet-encoding-vector,parquet-plugins/parquet-plugins-benchmarks
-am
+ # Spotless check uses palantir-java-format which relies on internal
javac APIs
+ # that are not available on all JDK versions (e.g. JDK 25+). Since
the formatting
+ # result is JDK-independent, running the check on JDK 17 alone is
sufficient.
+ SPOTLESS_ARGS=""
+ if [ "${{ matrix.java }}" != "17" ]; then
+ SPOTLESS_ARGS="-Dspotless.check.skip=true"
+ fi
+ ./mvnw verify --batch-mode -Pvector-plugins javadoc:javadoc
$SPOTLESS_ARGS -pl
parquet-plugins/parquet-encoding-vector,parquet-plugins/parquet-plugins-benchmarks
-am
diff --git a/.gitignore b/.gitignore
index ad049afc9..c02d0f222 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
*.class
.project
.classpath
+.factorypath
.settings
target
# Package Files #
@@ -20,5 +21,4 @@ target/
mvn_install.log
.vscode/*
.DS_Store
-.memsearch/
diff --git
a/parquet-plugins/parquet-encoding-vector/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPacking512VectorLE.java
b/parquet-plugins/parquet-encoding-vector/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPacking512VectorLE.java
index eb1690a4e..caf7686f1 100644
---
a/parquet-plugins/parquet-encoding-vector/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPacking512VectorLE.java
+++
b/parquet-plugins/parquet-encoding-vector/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPacking512VectorLE.java
@@ -177,7 +177,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(BYTE_SPECIES_64, in,
inPos, in.order());
+ ByteVector byteVector = fromByteBuffer(BYTE_SPECIES_64, in, inPos);
ShortVector tempRes = byteVector
.castShape(SHORT_SPECIES_512, 0)
.reinterpretAsBytes()
@@ -260,7 +260,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(BYTE_SPECIES, in,
inPos, in.order());
+ ByteVector byteVector = fromByteBuffer(BYTE_SPECIES, in, inPos);
ShortVector tempRes = byteVector
.castShape(LONG_SPECIES, 0)
.reinterpretAsBytes()
@@ -377,9 +377,8 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B128, in, inPos,
in.order())
- .castShape(S512, 0)
- .reinterpretAsBytes();
+ ByteVector byteVector =
+ fromByteBuffer(B128, in, inPos, inp_mask).castShape(S512,
0).reinterpretAsBytes();
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsShorts()
@@ -466,7 +465,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(BSPECIES, in, inPos,
in.order());
+ ByteVector byteVector = fromByteBuffer(BSPECIES, in, inPos);
ShortVector tempRes = byteVector
.castShape(ISPECIES, 0)
.reinterpretAsBytes()
@@ -582,9 +581,8 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B256, in, inPos,
in.order(), inp_mask)
- .castShape(S512, 0)
- .reinterpretAsBytes();
+ ByteVector byteVector =
+ fromByteBuffer(B256, in, inPos, inp_mask).castShape(S512,
0).reinterpretAsBytes();
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
@@ -705,9 +703,8 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B256, in, inPos,
in.order(), inp_mask)
- .castShape(S512, 0)
- .reinterpretAsBytes();
+ ByteVector byteVector =
+ fromByteBuffer(B256, in, inPos, inp_mask).castShape(S512,
0).reinterpretAsBytes();
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
@@ -827,9 +824,8 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B256, in, inPos,
in.order(), inp_mask)
- .castShape(S512, 0)
- .reinterpretAsBytes();
+ ByteVector byteVector =
+ fromByteBuffer(B256, in, inPos, inp_mask).castShape(S512,
0).reinterpretAsBytes();
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsShorts()
@@ -914,7 +910,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order());
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos);
byteVector
.castShape(ISPECIES, 0)
.lanewise(VectorOperators.AND, 255)
@@ -1004,7 +1000,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsShorts()
@@ -1084,7 +1080,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order());
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos);
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsShorts()
@@ -1194,7 +1190,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsShorts()
@@ -1280,7 +1276,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order());
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos);
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsShorts()
@@ -1388,7 +1384,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsShorts()
@@ -1512,7 +1508,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsShorts()
@@ -1630,7 +1626,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
ShortVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsShorts()
@@ -1703,7 +1699,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
ShortVector shortVector = byteVector.reinterpretAsShorts();
shortVector
.castShape(I512, 0)
@@ -1783,7 +1779,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -1866,7 +1862,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -1944,7 +1940,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -2022,7 +2018,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -2102,7 +2098,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -2182,7 +2178,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -2261,7 +2257,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -2332,7 +2328,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 =
byteVector.rearrange(perm_mask0).reinterpretAsInts().lanewise(VectorOperators.AND,
16777215);
tempRes1.intoArray(out, outPos, out_mask);
@@ -2407,7 +2403,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -2486,7 +2482,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -2603,7 +2599,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -2718,7 +2714,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -2832,7 +2828,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -2960,7 +2956,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -3089,7 +3085,7 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 = byteVector
.rearrange(perm_mask0)
.reinterpretAsInts()
@@ -3175,13 +3171,61 @@ public abstract class ByteBitPacking512VectorLE {
public final void unpackValuesUsingVector(
final ByteBuffer in, final int inPos, final int[] out, final int
outPos) {
- ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos,
in.order(), inp_mask);
+ ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask);
IntVector tempRes1 =
byteVector.rearrange(perm_mask0).reinterpretAsInts();
tempRes1.intoArray(out, outPos, out_mask);
}
}
+ // TODO Replace these helpers with ByteVector.fromMemorySegment(...) once
the project's minimum
+ // supported JDK is >= 22 (where java.lang.foreign.MemorySegment became a
permanent API per
+ // JEP 454). fromMemorySegment is the direct successor to
ByteVector.fromByteBuffer (which was
+ // removed after JDK 21) and is intrinsifiable by HotSpot for both heap and
direct buffers via
+ // jdk.internal.misc.ScopedMemoryAccess, eliminating the byte[] copy that
this fallback uses
+ // for direct ByteBuffers. Until then, the implementations below are
constrained to APIs
+ // available under --release 17. A multi-release JAR overlay
(src/main/java22) would let JDK
+ // 22+ runtimes pick up the fromMemorySegment path automatically; see
GH-3475 discussion.
+ private static ByteVector fromByteBuffer(VectorSpecies<Byte> species,
ByteBuffer input, int inPos) {
+ // Heap buffers are loaded directly from their backing array: no
ByteBuffer.duplicate(),
+ // no intermediate byte[] allocation, no extra copy. ByteVector.fromArray
is intrinsified by
+ // HotSpot and lowered to a single AVX-512 load.
+ if (input.hasArray()) {
+ return ByteVector.fromArray(species, input.array(), input.arrayOffset()
+ inPos);
+ }
+ // Off-heap (direct) buffers: fall back to copying species.length() bytes
into a scratch
+ // array. The original JDK 17 fast path used
ScopedMemoryAccess.loadFromByteBuffer to avoid
+ // the copy, but that intrinsic is not exposed; the alternative
ByteVector.fromMemorySegment
+ // is only available on JDK 19+ and cannot be called from --release 17
sources.
+ return ByteVector.fromArray(species, readInputBytes(input, inPos,
species.length()), 0);
+ }
+
+ private static ByteVector fromByteBuffer(
+ VectorSpecies<Byte> species, ByteBuffer input, int inPos,
VectorMask<Byte> mask) {
+ // Per JDK 17 ByteVector.fromByteBuffer(..., m) semantics, only masked-on
lanes must be in
+ // bounds. For heap buffers, fromArray handles this directly. For direct
buffers, we allocate
+ // a species.length()-sized array (satisfying fromArray's bounds-check
precondition) but only
+ // copy mask.trueCount() bytes — the minimum needed for the masked-on
lanes. The remaining
+ // positions stay zero and are ignored by the mask.
+ if (input.hasArray()) {
+ return ByteVector.fromArray(species, input.array(), input.arrayOffset()
+ inPos, mask);
+ }
+ byte[] bytes = new byte[species.length()];
+ int count = mask.trueCount();
+ for (int i = 0; i < count; i++) {
+ bytes[i] = input.get(inPos + i);
+ }
+ return ByteVector.fromArray(species, bytes, 0, mask);
+ }
+
+ private static byte[] readInputBytes(ByteBuffer input, int inPos, int
byteCount) {
+ byte[] bytes = new byte[byteCount];
+ ByteBuffer source = input.duplicate();
+ source.position(inPos);
+ source.get(bytes);
+ return bytes;
+ }
+
private static void notSupport() {
throw new RuntimeException(
"ByteBitPacking512VectorLE doesn't support the function, please use
ByteBitPackingLE!");
diff --git
a/parquet-plugins/parquet-encoding-vector/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBitPacking512VectorLE.java
b/parquet-plugins/parquet-encoding-vector/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBitPacking512VectorLE.java
index 3a510d269..73d972b34 100644
---
a/parquet-plugins/parquet-encoding-vector/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBitPacking512VectorLE.java
+++
b/parquet-plugins/parquet-encoding-vector/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBitPacking512VectorLE.java
@@ -41,8 +41,24 @@ public class TestByteBitPacking512VectorLE {
}
}
+ @Test
+ public void unpackValuesUsingVectorDirectByteBuffer() {
+ Assume.assumeTrue(ParquetReadRouter.getSupportVectorFromCPUFlags() ==
VectorSupport.VECTOR_512);
+ for (int i = 1; i <= 32; i++) {
+ unpackValuesUsingVectorBitWidthDirect(i);
+ }
+ }
+
+ @Test
+ public void unpackValuesUsingVectorReadOnlyByteBuffer() {
+ Assume.assumeTrue(ParquetReadRouter.getSupportVectorFromCPUFlags() ==
VectorSupport.VECTOR_512);
+ for (int i = 1; i <= 32; i++) {
+ unpackValuesUsingVectorBitWidthReadOnly(i);
+ }
+ }
+
private void unpackValuesUsingVectorBitWidth(int bitWidth) {
- try (Stream<int[]> intInputs = getRangeData(bitWidth)) {
+ try (Stream<int[]> intInputs = getRangeData(bitWidth, 8192)) {
intInputs.forEach(intInput -> {
int pack8Count = intInput.length / 8;
int byteOutputSize = pack8Count * bitWidth;
@@ -70,6 +86,64 @@ public class TestByteBitPacking512VectorLE {
}
}
+ private void unpackValuesUsingVectorBitWidthDirect(int bitWidth) {
+ // Use a smaller dataset to avoid OOM on CI; correctness of the vector
path is already
+ // exhaustively tested by unpackValuesUsingVectorBitWidth — here we only
verify that
+ // reading from a direct (off-heap) ByteBuffer produces the same result.
+ try (Stream<int[]> intInputs = getRangeData(bitWidth, 8192)) {
+ intInputs.forEach(intInput -> {
+ int pack8Count = intInput.length / 8;
+ int byteOutputSize = pack8Count * bitWidth;
+ byte[] byteOutput = new byte[byteOutputSize];
+ int[] output = new int[intInput.length];
+ int[] expected = new int[intInput.length];
+
+ BytePacker bytePacker = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ for (int i = 0; i < pack8Count; i++) {
+ bytePacker.pack8Values(intInput, 8 * i, byteOutput, bitWidth * i);
+ }
+
+ unpack8Values(bitWidth, byteOutput, expected);
+
+ // Direct (off-heap) ByteBuffer
+ ByteBuffer directBuffer = ByteBuffer.allocateDirect(byteOutputSize);
+ directBuffer.put(byteOutput);
+ directBuffer.flip();
+ unpackValuesUsingVectorByteBuffer(bitWidth, directBuffer, output);
+ assertArrayEquals(expected, output);
+ Arrays.fill(output, 0);
+ });
+ }
+ }
+
+ private void unpackValuesUsingVectorBitWidthReadOnly(int bitWidth) {
+ // Use a smaller dataset to avoid OOM on CI; correctness of the vector
path is already
+ // exhaustively tested by unpackValuesUsingVectorBitWidth — here we only
verify that
+ // reading from a read-only ByteBuffer produces the same result.
+ try (Stream<int[]> intInputs = getRangeData(bitWidth, 8192)) {
+ intInputs.forEach(intInput -> {
+ int pack8Count = intInput.length / 8;
+ int byteOutputSize = pack8Count * bitWidth;
+ byte[] byteOutput = new byte[byteOutputSize];
+ int[] output = new int[intInput.length];
+ int[] expected = new int[intInput.length];
+
+ BytePacker bytePacker = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ for (int i = 0; i < pack8Count; i++) {
+ bytePacker.pack8Values(intInput, 8 * i, byteOutput, bitWidth * i);
+ }
+
+ unpack8Values(bitWidth, byteOutput, expected);
+
+ // Read-only heap ByteBuffer (hasArray() returns false)
+ ByteBuffer readOnlyBuffer =
ByteBuffer.wrap(byteOutput).asReadOnlyBuffer();
+ unpackValuesUsingVectorByteBuffer(bitWidth, readOnlyBuffer, output);
+ assertArrayEquals(expected, output);
+ Arrays.fill(output, 0);
+ });
+ }
+ }
+
public void unpack8Values(int bitWidth, byte[] input, int[] output) {
BytePacker bytePacker = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
int len = input.length;
@@ -121,9 +195,7 @@ public class TestByteBitPacking512VectorLE {
}
}
- private Stream<int[]> getRangeData(int bitWidth) {
- int itemMax = 268435456;
-
+ private Stream<int[]> getRangeData(int bitWidth, int itemMax) {
long maxValue = getMaxValue(bitWidth);
long maxValueFilled = maxValue + 1;
int itemCount = (int) (maxValueFilled / itemMax);
@@ -134,39 +206,42 @@ public class TestByteBitPacking512VectorLE {
final int finalItemCount = itemCount;
- return IntStream.range(0, finalItemCount).mapToObj(i -> {
- int len;
- if ((i == finalItemCount - 1) && mode != 0) {
- len = mode;
- } else {
- len = itemMax;
- }
- if (len < 64) {
- len = 64;
- } else {
- len += 64;
- }
- int[] array = new int[len];
- int j = 0;
- while (j < len) {
- int value = j + i * itemMax;
- if (value > maxValue) {
- if (maxValue < Integer.MAX_VALUE) {
- value = (int) maxValue;
+ // Test the first and last chunks to cover both low-range and high-range
boundary values.
+ return IntStream.range(0, finalItemCount)
+ .filter(i -> i == 0 || i == finalItemCount - 1)
+ .mapToObj(i -> {
+ int len;
+ if ((i == finalItemCount - 1) && mode != 0) {
+ len = mode;
} else {
- value = Integer.MAX_VALUE;
+ len = itemMax;
}
- }
- if (value < 0) {
- if (bitWidth < 32) {
- value = value - Integer.MIN_VALUE;
+ if (len < 64) {
+ len = 64;
+ } else {
+ len += 64;
}
- }
- array[j] = value;
- j++;
- }
- return array;
- });
+ int[] array = new int[len];
+ int j = 0;
+ while (j < len) {
+ int value = j + i * itemMax;
+ if (value > maxValue) {
+ if (maxValue < Integer.MAX_VALUE) {
+ value = (int) maxValue;
+ } else {
+ value = Integer.MAX_VALUE;
+ }
+ }
+ if (value < 0) {
+ if (bitWidth < 32) {
+ value = value - Integer.MIN_VALUE;
+ }
+ }
+ array[j] = value;
+ j++;
+ }
+ return array;
+ });
}
private long getMaxValue(int bitWidth) {
