This is an automated email from the ASF dual-hosted git repository.
jackylee-ch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 32ce0f10c4 [VL] Take Arrow out of the bundle for Spark 4.x and drop
dead arrow-dataset (#12737)
32ce0f10c4 is described below
commit 32ce0f10c42e3cb150fce4056b35c9208530abab
Author: jackylee <[email protected]>
AuthorDate: Fri Aug 14 21:37:47 2026 +0800
[VL] Take Arrow out of the bundle for Spark 4.x and drop dead arrow-dataset
(#12737)
---
backends-velox/pom.xml | 63 ++++++++
.../org/apache/gluten/fs/ArrowFilesystemTest.java | 106 --------------
.../java/org/apache/gluten/fs/CsvWriteSupport.java | 52 -------
.../java/org/apache/gluten/fs/TestDataset.java | 117 ---------------
.../org/apache/gluten/fs/TestNativeDataset.java | 31 ----
dev/check-arrow-c-shading.sh | 162 ++++++++++++++++++---
gluten-arrow/pom.xml | 49 +++----
.../memory/arrow/pool/ArrowNativeMemoryPool.java | 75 ----------
.../arrow/pool/ArrowReservationListener.java | 41 ------
package/pom.xml | 27 +++-
pom.xml | 43 +++++-
11 files changed, 284 insertions(+), 482 deletions(-)
diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
index 2e3aa223d3..8f833020bb 100644
--- a/backends-velox/pom.xml
+++ b/backends-velox/pom.xml
@@ -79,6 +79,69 @@
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
+ <!--
+ Re-declared here because `provided` scope is not transitive: when
+ ${arrow.deps.scope} is `provided` (Spark 4.x), gluten-arrow's Arrow
+ dependencies do not reach this module's compile classpath on their own.
+ Under `compile` (Spark 3.x) these are redundant but harmless.
+
+ The allocator implementation (${arrow-memory.artifact}) is the exception:
+ it is never compiled against, only discovered on the classpath at
runtime,
+ so it uses ${arrow-memory.scope} (runtime on Spark 3.x, provided on 4.x)
+ rather than ${arrow.deps.scope}.
+ -->
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>${arrow-memory.artifact}</artifactId>
+ <version>${arrow.version}</version>
+ <scope>${arrow-memory.scope}</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-core</artifactId>
+ <version>${arrow.version}</version>
+ <scope>${arrow.deps.scope}</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ <version>${arrow.version}</version>
+ <scope>${arrow.deps.scope}</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
diff --git
a/backends-velox/src/test/java/org/apache/gluten/fs/ArrowFilesystemTest.java
b/backends-velox/src/test/java/org/apache/gluten/fs/ArrowFilesystemTest.java
deleted file mode 100644
index 5dd29856ac..0000000000
--- a/backends-velox/src/test/java/org/apache/gluten/fs/ArrowFilesystemTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.fs;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.primitives.Primitives;
-import org.apache.arrow.dataset.file.FileFormat;
-import org.apache.arrow.dataset.file.FileSystemDatasetFactory;
-import org.apache.arrow.dataset.jni.NativeMemoryPool;
-import org.apache.arrow.dataset.scanner.ScanOptions;
-import org.apache.arrow.util.AutoCloseables;
-import org.apache.arrow.vector.VectorLoader;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-import org.apache.arrow.vector.types.pojo.Schema;
-import org.junit.Assert;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-
-public class ArrowFilesystemTest extends TestNativeDataset {
-
- @ClassRule public static final TemporaryFolder TMP = new TemporaryFolder();
-
- private void checkParquetReadResult(
- Schema schema, String expectedJson, List<ArrowRecordBatch> actual)
throws IOException {
- final ObjectMapper json = new ObjectMapper();
- final Set<?> expectedSet = json.readValue(expectedJson, Set.class);
- final Set<List<Object>> actualSet = new HashSet<>();
- final int fieldCount = schema.getFields().size();
- try (VectorSchemaRoot vsr = VectorSchemaRoot.create(schema,
rootAllocator())) {
- VectorLoader loader = new VectorLoader(vsr);
- for (ArrowRecordBatch batch : actual) {
- System.out.println(batch.toString());
- loader.load(batch);
- int batchRowCount = vsr.getRowCount();
- for (int i = 0; i < batchRowCount; i++) {
- List<Object> row = new ArrayList<>();
- for (int j = 0; j < fieldCount; j++) {
- Object object = vsr.getVector(j).getObject(i);
- if (Primitives.isWrapperType(object.getClass())) {
- row.add(object);
- } else {
- row.add(object.toString());
- }
- }
- actualSet.add(row);
- }
- }
- }
- Assert.assertEquals(
- "Mismatched data read from Parquet, actual: " +
json.writeValueAsString(actualSet) + ";",
- expectedSet,
- actualSet);
- }
-
- @Test
- public void testBaseCsvRead() throws Exception {
- CsvWriteSupport writeSupport =
- CsvWriteSupport.writeTempFile(
- TMP.newFolder(), "Name,Language", "Juno,Java", "Peter,Python",
"Celin,C++");
- String expectedJsonUnordered =
- "[[\"Juno\", \"Java\"], [\"Peter\", \"Python\"], [\"Celin\",
\"C++\"]]";
- ScanOptions options = new ScanOptions(100);
- try (FileSystemDatasetFactory factory =
- new FileSystemDatasetFactory(
- rootAllocator(),
- NativeMemoryPool.getDefault(),
- FileFormat.CSV,
- writeSupport.getOutputURI())) {
- List<ArrowRecordBatch> datum = collectResultFromFactory(factory,
options);
- Schema schema = inferResultSchemaFromFactory(factory, options);
-
- assertScanBatchesProduced(factory, options);
- assertEquals(1, datum.size());
- assertEquals(2, schema.getFields().size());
- assertEquals("Name", schema.getFields().get(0).getName());
-
- checkParquetReadResult(schema, expectedJsonUnordered, datum);
-
- AutoCloseables.close(datum);
- }
- }
-}
diff --git
a/backends-velox/src/test/java/org/apache/gluten/fs/CsvWriteSupport.java
b/backends-velox/src/test/java/org/apache/gluten/fs/CsvWriteSupport.java
deleted file mode 100644
index bcc1cceb9a..0000000000
--- a/backends-velox/src/test/java/org/apache/gluten/fs/CsvWriteSupport.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.fs;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Random;
-
-public class CsvWriteSupport {
- private final URI uri;
- private final Random random = new Random();
-
- public CsvWriteSupport(File outputFolder) throws URISyntaxException {
- uri =
- new URI(
- "file",
- outputFolder.getPath() + File.separator + "generated-" +
random.nextLong() + ".csv",
- null);
- }
-
- public static CsvWriteSupport writeTempFile(File outputFolder, String...
values)
- throws URISyntaxException, IOException {
- CsvWriteSupport writer = new CsvWriteSupport(outputFolder);
- try (FileWriter addValues = new FileWriter(new File(writer.uri), true)) {
- for (Object value : values) {
- addValues.write(value + "\n");
- }
- }
- return writer;
- }
-
- public String getOutputURI() {
- return uri.toString();
- }
-}
diff --git a/backends-velox/src/test/java/org/apache/gluten/fs/TestDataset.java
b/backends-velox/src/test/java/org/apache/gluten/fs/TestDataset.java
deleted file mode 100644
index f2a1095bb2..0000000000
--- a/backends-velox/src/test/java/org/apache/gluten/fs/TestDataset.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.fs;
-
-import org.apache.arrow.dataset.scanner.ScanOptions;
-import org.apache.arrow.dataset.scanner.Scanner;
-import org.apache.arrow.dataset.source.Dataset;
-import org.apache.arrow.dataset.source.DatasetFactory;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.util.AutoCloseables;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
-import org.apache.arrow.vector.ipc.ArrowReader;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-import org.apache.arrow.vector.types.pojo.Schema;
-import org.junit.After;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Spliterator;
-import java.util.Spliterators;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-
-public abstract class TestDataset {
- private RootAllocator allocator = null;
-
- @Before
- public void setUp() {
- allocator = new RootAllocator(Long.MAX_VALUE);
- }
-
- @After
- public void tearDown() {
- allocator.close();
- }
-
- protected RootAllocator rootAllocator() {
- return allocator;
- }
-
- protected List<ArrowRecordBatch> collectResultFromFactory(
- DatasetFactory factory, ScanOptions options) {
- final Dataset dataset = factory.finish();
- final Scanner scanner = dataset.newScan(options);
- try {
- final List<ArrowRecordBatch> ret = collectTaskData(scanner);
- AutoCloseables.close(scanner, dataset);
- return ret;
- } catch (RuntimeException e) {
- throw e;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- protected List<ArrowRecordBatch> collectTaskData(Scanner scan) {
- try (ArrowReader reader = scan.scanBatches()) {
- List<ArrowRecordBatch> batches = new ArrayList<>();
- while (reader.loadNextBatch()) {
- VectorSchemaRoot root = reader.getVectorSchemaRoot();
- final VectorUnloader unloader = new VectorUnloader(root);
- batches.add(unloader.getRecordBatch());
- }
- return batches;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- protected Schema inferResultSchemaFromFactory(DatasetFactory factory,
ScanOptions options) {
- final Dataset dataset = factory.finish();
- final Scanner scanner = dataset.newScan(options);
- final Schema schema = scanner.schema();
- try {
- AutoCloseables.close(scanner, dataset);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- return schema;
- }
-
- protected <T> Stream<T> stream(Iterable<T> iterable) {
- return StreamSupport.stream(iterable.spliterator(), false);
- }
-
- protected <T> List<T> collect(Iterable<T> iterable) {
- return stream(iterable).collect(Collectors.toList());
- }
-
- protected <T> Stream<T> stream(Iterator<T> iterator) {
- return StreamSupport.stream(
- Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED),
false);
- }
-
- protected <T> List<T> collect(Iterator<T> iterator) {
- return stream(iterator).collect(Collectors.toList());
- }
-}
diff --git
a/backends-velox/src/test/java/org/apache/gluten/fs/TestNativeDataset.java
b/backends-velox/src/test/java/org/apache/gluten/fs/TestNativeDataset.java
deleted file mode 100644
index b06fa3c606..0000000000
--- a/backends-velox/src/test/java/org/apache/gluten/fs/TestNativeDataset.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.fs;
-
-import org.apache.arrow.dataset.scanner.ScanOptions;
-import org.apache.arrow.dataset.scanner.Scanner;
-import org.apache.arrow.dataset.source.Dataset;
-import org.apache.arrow.dataset.source.DatasetFactory;
-import org.junit.Assert;
-
-public abstract class TestNativeDataset extends TestDataset {
- protected void assertScanBatchesProduced(DatasetFactory factory, ScanOptions
options) {
- final Dataset dataset = factory.finish();
- final Scanner scanner = dataset.newScan(options);
- Assert.assertNotNull(scanner.scanBatches());
- }
-}
diff --git a/dev/check-arrow-c-shading.sh b/dev/check-arrow-c-shading.sh
index 620407603e..51504867d1 100755
--- a/dev/check-arrow-c-shading.sh
+++ b/dev/check-arrow-c-shading.sh
@@ -14,34 +14,65 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-# Verify the bundled gluten-velox jar's Arrow C-Data classes have method
-# signatures referencing the *unshaded* org.apache.arrow.memory.BufferAllocator
-# and org.apache.arrow.vector.* types — not the gluten-shaded copies.
+# Verify the bundled gluten-velox jar's Arrow C-Data classes reference the
+# *unshaded* Apache Arrow API — both in their method signatures and in their
+# constant pools.
#
# Background: org.apache.arrow.c.* must NOT be relocated (its native JNI binds
-# to the original class names), but its public API methods accept/return
-# org.apache.arrow.memory.* and org.apache.arrow.vector.* types. Those types
-# must therefore also stay unshaded in the bundle, otherwise the bundled
-# ArrowArrayStream/ArrowSchema get re-bound to the shaded BufferAllocator at
-# compile time and any caller passing a vanilla Apache Arrow allocator hits
-# `NoSuchMethodError`. See gluten#12225.
+# to the original class names), but it reaches into three other Arrow packages:
+# org.apache.arrow.memory.*, org.apache.arrow.vector.* (public signatures) and
+# org.apache.arrow.util.* (internal calls — Preconditions, AutoCloseables,
+# Collections2). All three must stay unshaded in the bundle:
+#
+# - a shaded *signature* type re-binds the bundled
ArrowArrayStream/ArrowSchema
+# so any caller passing a vanilla Apache Arrow allocator hits
+# `NoSuchMethodError` (gluten#12225);
+# - a shaded *constant-pool* reference is worse when Arrow is no longer
+# bundled at all: the shaded target does not exist anywhere on the
+# classpath and the call site throws `ClassNotFoundException`.
#
# Usage:
-# dev/check-arrow-c-shading.sh <path-to-gluten-velox-bundle.jar>
+# dev/check-arrow-c-shading.sh <path-to-gluten-velox-bundle.jar> \
+# [shade-package-name] [arrow-deps-scope]
+#
+# The shade package name defaults to org.apache.gluten.shaded and is passed by
+# package/pom.xml as ${gluten.shade.packageName}. Keep it parameterized: if
this
+# script hard-coded the prefix and the Maven property were ever changed, both
+# checks below would silently match nothing and the whole guard would pass
+# vacuously.
+#
+# The arrow-deps-scope is the value of the Maven property ${arrow.deps.scope}
+# and drives the bundle-content assertion:
+# - `compile` (Spark 3.x): gluten ships its own Arrow inside the bundle, so
+# arrow-memory / arrow-vector classes MUST be
+# present (the bundle is self-contained).
+# - `provided` (Spark 4.x): Arrow is expected to come from the Spark
+# distribution at runtime, so those packages MUST
+# NOT be inside the bundle (otherwise the bundle
+# has silently regressed to shipping its own
copy).
+# Any other value is not asserted against.
#
# Exit codes:
-# 0 — bundle is well-shaded (Arrow C-Data API uses public Apache Arrow types)
-# 1 — bundle is broken (Arrow C-Data API references gluten-shaded types)
+# 0 — bundle is well-shaded (Arrow C-Data API uses public Apache Arrow API)
+# 1 — bundle is broken (Arrow C-Data references gluten-shaded types, OR
+# Arrow content does not match the declared arrow-deps-scope)
# 2 — usage / setup error
set -euo pipefail
-JAR="${1:?usage: $0 <path-to-gluten-velox-bundle.jar>}"
+JAR="${1:?usage: $0 <path-to-gluten-velox-bundle.jar> [shade-package-name]
[arrow-deps-scope]}"
if [[ ! -f "$JAR" ]]; then
echo "error: jar not found: $JAR" >&2
exit 2
fi
+# Dotted form for javap signatures, slashed form for JVM internal names in
+# constant pools. `.` is escaped so the dotted form is a literal regex.
+SHADE_PACKAGE="${2:-org.apache.gluten.shaded}"
+SHADE_DOTS_RE="${SHADE_PACKAGE//./\\.}"
+SHADE_SLASHES="${SHADE_PACKAGE//.//}"
+ARROW_DEPS_SCOPE="${3:-}"
+
if ! command -v javap >/dev/null; then
echo "error: javap not found on PATH" >&2
exit 2
@@ -59,14 +90,22 @@ CLASSES=(
)
failures=0
+# Track whether the bundle actually contains any org.apache.arrow.c.* class.
+# When it doesn't, this jar is not the velox bundle (e.g. the intermediate
+# jar-plugin output built without the data-lake profiles that would pull Arrow
+# into the shade artifactSet), and the bundle-content assertion below has
+# nothing to say about it. The two shading checks are already SKIP-safe in
+# that case; the content assertion has to be too.
+cdata_present=0
for cls in "${CLASSES[@]}"; do
if ! unzip -p "$JAR" "${cls}.class" > "$WORKDIR/$(basename "$cls").class"
2>/dev/null; then
echo " SKIP $cls (not in bundle)"
continue
fi
+ cdata_present=1
signatures=$(javap -p "$WORKDIR/$(basename "$cls").class" 2>/dev/null ||
true)
- # Any method signature mentioning the gluten-shaded Arrow path is the bug.
- bad=$(echo "$signatures" | grep -E
"org\.apache\.gluten\.shaded\.org\.apache\.arrow\.(memory|vector)\." || true)
+ # Any method signature mentioning the shaded Arrow path is the bug.
+ bad=$(echo "$signatures" | grep -E
"${SHADE_DOTS_RE}\.org\.apache\.arrow\.(memory|vector)\." || true)
if [[ -n "$bad" ]]; then
echo " FAIL $cls — public API references gluten-shaded Arrow types:"
echo "$bad" | sed 's/^/ /'
@@ -76,12 +115,97 @@ for cls in "${CLASSES[@]}"; do
fi
done
+# Second check: no class under org/apache/arrow/c/ may *call* a shaded Arrow
+# class. Signatures alone miss org.apache.arrow.util.Preconditions & friends,
+# which are invoked from constructors but never appear in a descriptor.
+#
+# Both org.apache.arrow.c.* and org.apache.arrow.c.jni.* are excluded from
+# relocation in package/pom.xml, so both are scanned. The jni subpackage is
+# named explicitly rather than relying on `unzip` treating `c/*` as recursive —
+# that is implementation-defined, and the existence check below would otherwise
+# see an empty top level and skip the scan entirely.
+#
+# The name pattern covers every character legal in a JVM internal name after
the
+# package prefix: identifier chars (letters, digits, `_`, `$`), `/` for nested
+# packages, and `-` for the synthetic `package-info` / `module-info` entries.
+mkdir -p "$WORKDIR/all"
+unzip -qo "$JAR" 'org/apache/arrow/c/*' 'org/apache/arrow/c/jni/*' \
+ -d "$WORKDIR/all" 2>/dev/null || true
+if compgen -G "$WORKDIR/all/org/apache/arrow/c/**/*.class" > /dev/null ||
+ compgen -G "$WORKDIR/all/org/apache/arrow/c/*.class" > /dev/null; then
+ cdata_present=1
+ refs=$(grep -rahoE "${SHADE_SLASHES}/org/apache/arrow/[a-zA-Z0-9_$/-]+" \
+ "$WORKDIR/all/org/apache/arrow/c" 2>/dev/null | sort -u || true)
+ if [[ -n "$refs" ]]; then
+ echo " FAIL org/apache/arrow/c/** — calls into gluten-shaded Arrow:"
+ echo "$refs" | sed 's/^/ /'
+ failures=$((failures + 1))
+ else
+ echo " OK org/apache/arrow/c/** constant pools"
+ fi
+fi
+
+# Third check: the bundle's Arrow content must match ${arrow.deps.scope}.
+# This is the regression guard for #12737 — if any Arrow dependency ever slips
+# from `provided`/`runtime` back to `compile` on a Spark 4.x profile, the
memory
+# and vector packages silently re-enter the bundle, undoing the size win and
+# re-introducing the Spark-vs-gluten Arrow version conflict. Assert directly on
+# the jar contents so the mistake fails the build instead of shipping.
+#
+# Gated on arrow-c-data being present. Whether Arrow lands in the jar at all is
+# a function of the dependency closure and the shade artifactSet, not just of
+# ${arrow.deps.scope}: `mvn install -Pspark-3.5 -Pbackends-velox` (no data-lake
+# profiles) produces an intermediate jar with no Arrow whatsoever. arrow-c-data
+# is bundled on every profile precisely because Spark never ships it, so its
+# presence is the reliable marker for "this is the velox bundle". Without that
+# gate, the `compile` branch below fires on jars that were never meant to carry
+# Arrow at all.
+if [[ -n "$ARROW_DEPS_SCOPE" && "$cdata_present" -eq 0 ]]; then
+ echo " SKIP bundle-content assertion (no org.apache.arrow.c.* in jar —"
+ echo " not a velox bundle, so its Arrow content is not asserted)"
+elif [[ -n "$ARROW_DEPS_SCOPE" ]]; then
+ # arrow-memory-core / arrow-vector classes, excluding the always-bundled
+ # org.apache.arrow.c.* (arrow-c-data) which Spark never ships.
+ arrow_impl=$(unzip -l "$JAR" 2>/dev/null \
+ | grep -oE "org/apache/arrow/(memory|vector)/[^ ]*\.class" | sort -u ||
true)
+ impl_count=$(printf '%s' "$arrow_impl" | grep -c . || true)
+ case "$ARROW_DEPS_SCOPE" in
+ provided)
+ if [[ "$impl_count" -gt 0 ]]; then
+ echo " FAIL bundle content — arrow.deps.scope=provided but the bundle"
+ echo " still ships $impl_count arrow-memory/arrow-vector
class(es):"
+ printf '%s\n' "$arrow_impl" | head -5 | sed 's/^/ /'
+ echo " A dependency likely regressed to compile scope. Arrow must
come"
+ echo " from the Spark distribution at runtime on Spark 4.x."
+ failures=$((failures + 1))
+ else
+ echo " OK bundle carries no arrow-memory/arrow-vector
(scope=provided)"
+ fi
+ ;;
+ compile)
+ if [[ "$impl_count" -eq 0 ]]; then
+ echo " FAIL bundle content — arrow.deps.scope=compile but the bundle"
+ echo " ships no arrow-memory/arrow-vector classes; the
self-contained"
+ echo " bundle is incomplete and will fail to allocate Arrow
buffers."
+ failures=$((failures + 1))
+ else
+ echo " OK bundle carries arrow-memory/arrow-vector (scope=compile)"
+ fi
+ ;;
+ *)
+ echo " SKIP bundle-content assertion (unrecognized
arrow.deps.scope='$ARROW_DEPS_SCOPE')"
+ ;;
+ esac
+fi
+
if (( failures > 0 )); then
echo
- echo "Bundle has $failures Arrow C-Data class(es) with shaded API types."
- echo "See gluten#12225 for context. Update package/pom.xml's"
- echo "<relocation org.apache.arrow> excludes to also exclude"
- echo "org.apache.arrow.memory.** and org.apache.arrow.vector.**."
+ echo "Bundle has $failures Arrow shading/content problem(s)."
+ echo "For shading failures, see gluten#12225 and update package/pom.xml's"
+ echo "<relocation org.apache.arrow> excludes so every package reachable"
+ echo "from org.apache.arrow.c stays unshaded (memory, vector, util)."
+ echo "For content failures, see gluten#12737 and check each Arrow"
+ echo "dependency's <scope> against \${arrow.deps.scope} for this profile."
exit 1
fi
diff --git a/gluten-arrow/pom.xml b/gluten-arrow/pom.xml
index d08d997ed2..435e967685 100644
--- a/gluten-arrow/pom.xml
+++ b/gluten-arrow/pom.xml
@@ -89,13 +89,23 @@
<groupId>org.apache.arrow</groupId>
<artifactId>${arrow-memory.artifact}</artifactId>
<version>${arrow.version}</version>
- <scope>runtime</scope>
+ <scope>${arrow-memory.scope}</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-core</artifactId>
<version>${arrow.version}</version>
- <scope>compile</scope>
+ <scope>${arrow.deps.scope}</scope>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
@@ -111,6 +121,7 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<version>${arrow.version}</version>
+ <scope>${arrow.deps.scope}</scope>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
@@ -132,37 +143,19 @@
<version>${arrow.version}</version>
<scope>compile</scope>
<exclusions>
+ <!--
+ arrow-vector / arrow-memory-core are declared above with
+ ${arrow.deps.scope}; keep them out of this artifact's transitive
+ closure so they cannot re-enter at compile scope and land in the
+ shaded bundle when the scope is `provided`.
+ -->
<exclusion>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
</exclusion>
<exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.arrow</groupId>
- <artifactId>arrow-dataset</artifactId>
- <version>${arrow.version}</version>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-buffer</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
diff --git
a/gluten-arrow/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java
b/gluten-arrow/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java
deleted file mode 100644
index f1aa6149f9..0000000000
---
a/gluten-arrow/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.memory.arrow.pool;
-
-import org.apache.arrow.dataset.jni.NativeMemoryPool;
-import org.apache.spark.task.TaskResource;
-import org.apache.spark.task.TaskResources;
-import org.apache.spark.util.Utils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ArrowNativeMemoryPool implements TaskResource {
- private static final Logger LOGGER =
LoggerFactory.getLogger(ArrowNativeMemoryPool.class);
-
- private final NativeMemoryPool arrowPool;
- private final ArrowReservationListener listener;
-
- public ArrowNativeMemoryPool() {
- listener = new ArrowReservationListener(TaskResources.getSharedUsage());
- arrowPool = NativeMemoryPool.createListenable(listener);
- }
-
- public static NativeMemoryPool arrowPool(String name) {
- if (!TaskResources.inSparkTask()) {
- throw new IllegalStateException("This method must be called in a Spark
task.");
- }
- String id = "ArrowNativeMemoryPool:" + name;
- return TaskResources.addResourceIfNotRegistered(id, () ->
createArrowNativeMemoryPool(name))
- .getArrowPool();
- }
-
- private static ArrowNativeMemoryPool createArrowNativeMemoryPool(String
name) {
- return new ArrowNativeMemoryPool();
- }
-
- @Override
- public void release() throws Exception {
- if (arrowPool.getBytesAllocated() != 0) {
- LOGGER.warn(
- String.format(
- "Arrow pool still reserved non-zero bytes, "
- + "which may cause memory leak, size: %s. ",
- Utils.bytesToString(arrowPool.getBytesAllocated())));
- }
- arrowPool.close();
- }
-
- @Override
- public int priority() {
- return 10;
- }
-
- @Override
- public String resourceName() {
- return "arrow_mem";
- }
-
- public NativeMemoryPool getArrowPool() {
- return arrowPool;
- }
-}
diff --git
a/gluten-arrow/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowReservationListener.java
b/gluten-arrow/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowReservationListener.java
deleted file mode 100644
index 2e1a254453..0000000000
---
a/gluten-arrow/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowReservationListener.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.memory.arrow.pool;
-
-import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
-
-public class ArrowReservationListener implements
org.apache.arrow.dataset.jni.ReservationListener {
- private final SimpleMemoryUsageRecorder sharedUsage; // shared task metrics
-
- public ArrowReservationListener(SimpleMemoryUsageRecorder recorder) {
- this.sharedUsage = recorder;
- }
-
- @Override
- public void reserve(long size) {
- synchronized (this) {
- sharedUsage.inc(size);
- }
- }
-
- @Override
- public void unreserve(long size) {
- synchronized (this) {
- sharedUsage.inc(-size);
- }
- }
-}
diff --git a/package/pom.xml b/package/pom.xml
index 513a3352bd..cee31a8ecf 100644
--- a/package/pom.xml
+++ b/package/pom.xml
@@ -122,22 +122,31 @@
<pattern>org.apache.arrow</pattern>
<shadedPattern>${gluten.shade.packageName}.org.apache.arrow</shadedPattern>
<!--
- arrow's C and dataset wrappers refer to the original class
- path, so they must not be relocated. Their public APIs also
- take and return org.apache.arrow.memory.* and
- org.apache.arrow.vector.* types, so those packages must
also
- stay unshaded — otherwise the bundled (unshaded)
- ArrowArrayStream/ArrowSchema get compiled against the
- relocated BufferAllocator/VectorSchemaRoot, producing
+ arrow's C wrapper refers to the original class path, so it
+ must not be relocated. Its public APIs also take and return
+ org.apache.arrow.memory.* and org.apache.arrow.vector.*
+ types, so those packages must also stay unshaded —
+ otherwise the bundled (unshaded)
ArrowArrayStream/ArrowSchema
+ get compiled against the relocated
+ BufferAllocator/VectorSchemaRoot, producing
`NoSuchMethodError` for any caller passing a vanilla
Apache Arrow allocator. See #12225.
+
+ org.apache.arrow.util.** is the fourth and last package
+ org.apache.arrow.c reaches: Preconditions.checkNotNull from
+ the ArrowArray/ArrowSchema constructors, plus
AutoCloseables
+ and Collections2 from the import/export paths. It is not
part
+ of the public signature so it survived #12225, but the call
+ sites live in the constant pool of the unshaded classes all
+ the same, and there is no unshaded copy to bind to once
+ arrow-memory-core stops being bundled.
-->
<excludes>
<exclude>org.apache.arrow.c.*</exclude>
<exclude>org.apache.arrow.c.jni.*</exclude>
<exclude>org.apache.arrow.memory.**</exclude>
<exclude>org.apache.arrow.vector.**</exclude>
- <exclude>org.apache.arrow.dataset.**</exclude>
+ <exclude>org.apache.arrow.util.**</exclude>
</excludes>
</relocation>
<relocation>
@@ -341,6 +350,8 @@
<executable>${project.basedir}/../dev/check-arrow-c-shading.sh</executable>
<arguments>
<argument>${project.build.directory}/${project.build.finalName}.jar</argument>
+ <argument>${gluten.shade.packageName}</argument>
+ <argument>${arrow.deps.scope}</argument>
</arguments>
</configuration>
</execution>
diff --git a/pom.xml b/pom.xml
index 6ea3e03b12..69adaaed86 100644
--- a/pom.xml
+++ b/pom.xml
@@ -82,6 +82,31 @@
<uniffle.version>0.10.0</uniffle.version>
<arrow.version>15.0.0</arrow.version>
<arrow-memory.artifact>arrow-memory-unsafe</arrow-memory.artifact>
+ <!--
+ Scope of gluten's Arrow dependencies. `compile` means gluten ships its
own
+ Arrow inside the bundle jar; `provided` means the Spark distribution on
the
+ runtime classpath owns Arrow and gluten only compiles against it. Spark
4.x
+ already ships an Arrow new enough for gluten, so those profiles switch to
+ `provided`.
+ -->
+ <arrow.deps.scope>compile</arrow.deps.scope>
+ <!--
+ Scope of the Arrow allocator *implementation* (${arrow-memory.artifact}).
+ Gluten never compiles against it — `CheckAllocator` discovers it by
+ scanning the classpath at runtime — so it stays out of the compile scope
+ and off downstream modules' compile classpaths. `runtime` still lands it
+ in the shaded bundle; Spark 4.x overrides this with `provided` so Spark's
+ own allocator is used instead.
+ -->
+ <arrow-memory.scope>runtime</arrow-memory.scope>
+ <!--
+ groupId used in the Arrow <exclusion>s on the Spark dependencies below.
+ Set to `org.apache.arrow` (the real groupId) the exclusions apply, so
+ gluten's own bundled Arrow wins. Profiles that use `provided` Arrow
+ override this with a groupId that matches nothing, letting Spark's Arrow
+ through instead.
+ -->
+
<spark.arrow.exclusion.groupId>org.apache.arrow</spark.arrow.exclusion.groupId>
<hadoop.version>2.7.4</hadoop.version>
<antlr4.version>4.9.3</antlr4.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -186,7 +211,7 @@
<scope>provided</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.arrow</groupId>
+ <groupId>${spark.arrow.exclusion.groupId}</groupId>
<artifactId>arrow-vector</artifactId>
</exclusion>
<exclusion>
@@ -242,7 +267,7 @@
<scope>provided</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.arrow</groupId>
+ <groupId>${spark.arrow.exclusion.groupId}</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
@@ -339,7 +364,7 @@
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.arrow</groupId>
+ <groupId>${spark.arrow.exclusion.groupId}</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
@@ -352,7 +377,7 @@
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.arrow</groupId>
+ <groupId>${spark.arrow.exclusion.groupId}</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
@@ -1302,6 +1327,10 @@
<log4j.version>2.24.3</log4j.version>
<commons-lang3.version>3.17.0</commons-lang3.version>
<arrow.version>18.1.0</arrow.version>
+ <arrow.deps.scope>provided</arrow.deps.scope>
+ <arrow-memory.scope>provided</arrow-memory.scope>
+ <!-- Matches no groupId: let Spark's own Arrow through. -->
+ <spark.arrow.exclusion.groupId>none</spark.arrow.exclusion.groupId>
<scala.compiler.version>4.9.2</scala.compiler.version>
</properties>
<dependencies>
@@ -1380,7 +1409,11 @@
<slf4j.version>2.0.16</slf4j.version>
<log4j.version>2.24.3</log4j.version>
<commons-lang3.version>3.17.0</commons-lang3.version>
- <arrow.version>18.1.0</arrow.version>
+ <arrow.version>18.3.0</arrow.version>
+ <arrow.deps.scope>provided</arrow.deps.scope>
+ <arrow-memory.scope>provided</arrow-memory.scope>
+ <!-- Matches no groupId: let Spark's own Arrow through. -->
+ <spark.arrow.exclusion.groupId>none</spark.arrow.exclusion.groupId>
<scala.compiler.version>4.9.5</scala.compiler.version>
</properties>
<dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
