This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new bc4dc12ca [CELEBORN-2239] Support Spark 4.1
bc4dc12ca is described below
commit bc4dc12cae0fee63b5951658f2c89dc5725eec22
Author: SteNicholas <[email protected]>
AuthorDate: Fri Feb 27 19:59:26 2026 +0800
[CELEBORN-2239] Support Spark 4.1
### What changes were proposed in this pull request?
Support Spark 4.1.
### Why are the changes needed?
Spark 4.1.1 has already released, which release notes refer to [Spark
Release 4.1.1](https://spark.apache.org/releases/spark-release-4.1.1.html).
### Does this PR resolve a correctness bug?
No.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI.
Closes #3571 from SteNicholas/CELEBORN-2239.
Authored-by: SteNicholas <[email protected]>
Signed-off-by: SteNicholas <[email protected]>
---
.github/workflows/deps.yml | 1 +
.github/workflows/maven.yml | 1 +
.github/workflows/sbt.yml | 3 ++
README.md | 3 +-
build/make-distribution.sh | 2 +
build/release/release.sh | 2 +-
dev/deps/dependencies-client-spark-4.1 | 83 ++++++++++++++++++++++++++++++++++
dev/reformat | 1 +
docs/developers/sbt.md | 1 +
pom.xml | 19 ++++++++
project/CelebornBuild.scala | 21 ++++++++-
tests/spark-it/pom.xml | 18 ++++++++
12 files changed, 152 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml
index f40e194a5..efa28a2eb 100644
--- a/.github/workflows/deps.yml
+++ b/.github/workflows/deps.yml
@@ -129,6 +129,7 @@ jobs:
matrix:
module:
- 'spark-4.0'
+ - 'spark-4.1'
steps:
- uses: actions/checkout@v4
- name: Setup JDK 17
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index 513e3a891..265e6d5b9 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -156,6 +156,7 @@ jobs:
- 17
spark:
- '4.0'
+ - '4.1'
shuffle-plugin-class:
- 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
steps:
diff --git a/.github/workflows/sbt.yml b/.github/workflows/sbt.yml
index b4e00c0d2..4daff4409 100644
--- a/.github/workflows/sbt.yml
+++ b/.github/workflows/sbt.yml
@@ -203,6 +203,7 @@ jobs:
matrix:
spark:
- '4.0'
+ - '4.1'
java:
- 17
scala-binary:
@@ -212,6 +213,8 @@ jobs:
include:
- spark: '4.0'
scala: '2.13.16'
+ - spark: '4.1'
+ scala: '2.13.17'
steps:
- uses: actions/checkout@v4
- name: Setup JDK ${{ matrix.java }}
diff --git a/README.md b/README.md
index 6eb54c972..bdf66e027 100644
--- a/README.md
+++ b/README.md
@@ -46,7 +46,7 @@ Celeborn worker's slot count decreases when a partition is
allocated and increme
Build Celeborn via `make-distribution.sh`:
```shell
-./build/make-distribution.sh
-Pspark-2.4/-Pspark-3.0/-Pspark-3.1/-Pspark-3.2/-Pspark-3.3/-Pspark-3.4/-Pspark-3.5/-Pspark-4.0/-Pflink-1.16/-Pflink-1.17/-Pflink-1.18/-Pflink-1.19/-Pflink-1.20/-Pflink-2.0/-Pflink-2.1/-Pflink-2.2/-Pmr
+./build/make-distribution.sh
-Pspark-2.4/-Pspark-3.0/-Pspark-3.1/-Pspark-3.2/-Pspark-3.3/-Pspark-3.4/-Pspark-3.5/-Pspark-4.0/-Pspark-4.1/-Pflink-1.16/-Pflink-1.17/-Pflink-1.18/-Pflink-1.19/-Pflink-1.20/-Pflink-2.0/-Pflink-2.1/-Pflink-2.2/-Pmr
```
Package `apache-celeborn-${project.version}-bin.tgz` will be generated.
@@ -63,6 +63,7 @@ Package `apache-celeborn-${project.version}-bin.tgz` will be
generated.
| Spark 3.4 | ❌ | ✔ | ✔ |
✔ | ✔ | ✔ | ✔
|
| Spark 3.5 | ❌ | ✔ | ✔ |
✔ | ✔ | ✔ | ✔
|
| Spark 4.0 | ❌ | ❌ | ❌ |
❌ | ❌ | ❌ | ✔
|
+| Spark 4.1 | ❌ | ❌ | ❌ |
❌ | ❌ | ❌ | ✔
|
| Flink 1.16 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Flink 1.17 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Flink 1.18 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
diff --git a/build/make-distribution.sh b/build/make-distribution.sh
index f0e659c5d..e160b758d 100755
--- a/build/make-distribution.sh
+++ b/build/make-distribution.sh
@@ -356,6 +356,7 @@ if [ "$SBT_ENABLED" == "true" ]; then
sbt_build_client -Pspark-3.5
export JAVA_HOME=$JAVA17_HOME
sbt_build_client -Pspark-4.0
+ sbt_build_client -Pspark-4.1
export JAVA_HOME=$JAVA8_HOME
sbt_build_client -Pflink-1.16
sbt_build_client -Pflink-1.17
@@ -399,6 +400,7 @@ else
build_spark_client -Pspark-3.5
export JAVA_HOME=$JAVA17_HOME
build_spark_client -Pspark-4.0
+ build_spark_client -Pspark-4.1
export JAVA_HOME=$JAVA8_HOME
build_flink_client -Pflink-1.16
build_flink_client -Pflink-1.17
diff --git a/build/release/release.sh b/build/release/release.sh
index 019f6383f..b3954c333 100755
--- a/build/release/release.sh
+++ b/build/release/release.sh
@@ -115,7 +115,7 @@ upload_nexus_staging() {
export JAVA_HOME=$JAVA17_HOME
echo "Deploying celeborn-client-spark-4-shaded_2.13"
- ${PROJECT_DIR}/build/sbt -Pspark-4.0
"clean;celeborn-client-spark-4-shaded/publishSigned"
+ ${PROJECT_DIR}/build/sbt -Pspark-4.1
"clean;celeborn-client-spark-4-shaded/publishSigned"
export JAVA_HOME=$JAVA8_HOME
echo "Deploying celeborn-client-flink-1.16-shaded_2.12"
diff --git a/dev/deps/dependencies-client-spark-4.1
b/dev/deps/dependencies-client-spark-4.1
new file mode 100644
index 000000000..8ad85bec0
--- /dev/null
+++ b/dev/deps/dependencies-client-spark-4.1
@@ -0,0 +1,83 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar
+commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-io/2.17.0//commons-io-2.17.0.jar
+commons-lang3/3.17.0//commons-lang3-3.17.0.jar
+commons-logging/1.1.3//commons-logging-1.1.3.jar
+failureaccess/1.0.2//failureaccess-1.0.2.jar
+guava/33.1.0-jre//guava-33.1.0-jre.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
+jackson-annotations/2.15.3//jackson-annotations-2.15.3.jar
+jackson-core/2.15.3//jackson-core-2.15.3.jar
+jackson-databind/2.15.3//jackson-databind-2.15.3.jar
+jackson-module-scala_2.13/2.15.3//jackson-module-scala_2.13-2.15.3.jar
+jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
+jsr305/1.3.9//jsr305-1.3.9.jar
+jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
+leveldbjni-all/1.8//leveldbjni-all-1.8.jar
+lz4-java/1.8.0//lz4-java-1.8.0.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
+netty-all/4.2.10.Final//netty-all-4.2.10.Final.jar
+netty-buffer/4.2.10.Final//netty-buffer-4.2.10.Final.jar
+netty-codec-base/4.2.10.Final//netty-codec-base-4.2.10.Final.jar
+netty-codec-classes-quic/4.2.10.Final//netty-codec-classes-quic-4.2.10.Final.jar
+netty-codec-compression/4.2.10.Final//netty-codec-compression-4.2.10.Final.jar
+netty-codec-dns/4.2.10.Final//netty-codec-dns-4.2.10.Final.jar
+netty-codec-http/4.2.10.Final//netty-codec-http-4.2.10.Final.jar
+netty-codec-http2/4.2.10.Final//netty-codec-http2-4.2.10.Final.jar
+netty-codec-http3/4.2.10.Final//netty-codec-http3-4.2.10.Final.jar
+netty-codec-marshalling/4.2.10.Final//netty-codec-marshalling-4.2.10.Final.jar
+netty-codec-native-quic/4.2.10.Final/linux-aarch_64/netty-codec-native-quic-4.2.10.Final-linux-aarch_64.jar
+netty-codec-native-quic/4.2.10.Final/linux-x86_64/netty-codec-native-quic-4.2.10.Final-linux-x86_64.jar
+netty-codec-native-quic/4.2.10.Final/osx-aarch_64/netty-codec-native-quic-4.2.10.Final-osx-aarch_64.jar
+netty-codec-native-quic/4.2.10.Final/osx-x86_64/netty-codec-native-quic-4.2.10.Final-osx-x86_64.jar
+netty-codec-native-quic/4.2.10.Final/windows-x86_64/netty-codec-native-quic-4.2.10.Final-windows-x86_64.jar
+netty-codec-protobuf/4.2.10.Final//netty-codec-protobuf-4.2.10.Final.jar
+netty-codec-socks/4.2.10.Final//netty-codec-socks-4.2.10.Final.jar
+netty-codec/4.2.10.Final//netty-codec-4.2.10.Final.jar
+netty-common/4.2.10.Final//netty-common-4.2.10.Final.jar
+netty-handler-proxy/4.2.10.Final//netty-handler-proxy-4.2.10.Final.jar
+netty-handler/4.2.10.Final//netty-handler-4.2.10.Final.jar
+netty-resolver-dns/4.2.10.Final//netty-resolver-dns-4.2.10.Final.jar
+netty-resolver/4.2.10.Final//netty-resolver-4.2.10.Final.jar
+netty-transport-classes-epoll/4.2.10.Final//netty-transport-classes-epoll-4.2.10.Final.jar
+netty-transport-classes-io_uring/4.2.10.Final//netty-transport-classes-io_uring-4.2.10.Final.jar
+netty-transport-classes-kqueue/4.2.10.Final//netty-transport-classes-kqueue-4.2.10.Final.jar
+netty-transport-native-epoll/4.2.10.Final/linux-aarch_64/netty-transport-native-epoll-4.2.10.Final-linux-aarch_64.jar
+netty-transport-native-epoll/4.2.10.Final/linux-riscv64/netty-transport-native-epoll-4.2.10.Final-linux-riscv64.jar
+netty-transport-native-epoll/4.2.10.Final/linux-x86_64/netty-transport-native-epoll-4.2.10.Final-linux-x86_64.jar
+netty-transport-native-io_uring/4.2.10.Final/linux-aarch_64/netty-transport-native-io_uring-4.2.10.Final-linux-aarch_64.jar
+netty-transport-native-io_uring/4.2.10.Final/linux-riscv64/netty-transport-native-io_uring-4.2.10.Final-linux-riscv64.jar
+netty-transport-native-io_uring/4.2.10.Final/linux-x86_64/netty-transport-native-io_uring-4.2.10.Final-linux-x86_64.jar
+netty-transport-native-kqueue/4.2.10.Final/osx-aarch_64/netty-transport-native-kqueue-4.2.10.Final-osx-aarch_64.jar
+netty-transport-native-kqueue/4.2.10.Final/osx-x86_64/netty-transport-native-kqueue-4.2.10.Final-osx-x86_64.jar
+netty-transport-native-unix-common/4.2.10.Final//netty-transport-native-unix-common-4.2.10.Final.jar
+netty-transport/4.2.10.Final//netty-transport-4.2.10.Final.jar
+paranamer/2.8.3//paranamer-2.8.3.jar
+paranamer/2.8//paranamer-2.8.jar
+protobuf-java/3.25.5//protobuf-java-3.25.5.jar
+scala-library/2.13.17//scala-library-2.13.17.jar
+scala-reflect/2.13.17//scala-reflect-2.13.17.jar
+slf4j-api/1.7.36//slf4j-api-1.7.36.jar
+snakeyaml/2.2//snakeyaml-2.2.jar
+snappy-java/1.1.10.5//snappy-java-1.1.10.5.jar
+zstd-jni/1.5.7-6//zstd-jni-1.5.7-6.jar
diff --git a/dev/reformat b/dev/reformat
index 5c702536a..b6fe643e0 100755
--- a/dev/reformat
+++ b/dev/reformat
@@ -35,6 +35,7 @@ else
${PROJECT_DIR}/build/mvn spotless:apply -Pspark-3.3
${PROJECT_DIR}/build/mvn spotless:apply -Pspark-3.5
${PROJECT_DIR}/build/mvn spotless:apply -Pspark-4.0
+ ${PROJECT_DIR}/build/mvn spotless:apply -Pspark-4.1
${PROJECT_DIR}/build/mvn spotless:apply -Paws
${PROJECT_DIR}/build/mvn spotless:apply -Paliyun
${PROJECT_DIR}/build/mvn spotless:apply -Pmr
diff --git a/docs/developers/sbt.md b/docs/developers/sbt.md
index 390e07646..60617f86a 100644
--- a/docs/developers/sbt.md
+++ b/docs/developers/sbt.md
@@ -36,6 +36,7 @@ The following table indicates the compatibility of Celeborn
Spark and Flink clie
| Spark 3.4 | ❌ | ✔ | ✔ |
✔ | ✔ | ✔ | ✔
|
| Spark 3.5 | ❌ | ✔ | ✔ |
✔ | ✔ | ✔ | ✔
|
| Spark 4.0 | ❌ | ❌ | ❌ |
❌ | ❌ | ❌ | ✔
|
+| Spark 4.1 | ❌ | ❌ | ❌ |
❌ | ❌ | ❌ | ✔
|
| Flink 1.16 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Flink 1.17 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Flink 1.18 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
diff --git a/pom.xml b/pom.xml
index d158d50e8..5bf67c3ba 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1673,6 +1673,25 @@
</properties>
</profile>
+ <profile>
+ <id>spark-4.1</id>
+ <modules>
+ <module>client-spark/common</module>
+ <module>client-spark/spark-3</module>
+ <module>client-spark/spark-3-columnar-common</module>
+ <module>client-spark/spark-4-columnar-shuffle</module>
+ <module>client-spark/spark-4-shaded</module>
+ <module>tests/spark-it</module>
+ </modules>
+ <properties>
+ <lz4-java.version>1.8.0</lz4-java.version>
+ <scala.version>2.13.17</scala.version>
+ <scala.binary.version>2.13</scala.binary.version>
+ <spark.version>4.1.1</spark.version>
+ <zstd-jni.version>1.5.7-6</zstd-jni.version>
+ </properties>
+ </profile>
+
<profile>
<id>jdk-8</id>
<activation>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index 0d781fafb..45b525f2f 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -317,7 +317,8 @@ object CelebornCommonSettings {
val SCALA_2_13_5 = "2.13.5"
val SCALA_2_13_8 = "2.13.8"
val SCALA_2_13_16 = "2.13.16"
- val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15,
SCALA_2_12_17, SCALA_2_12_18, SCALA_2_13_5, SCALA_2_13_8, SCALA_2_13_16)
+ val SCALA_2_13_17 = "2.13.17"
+ val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15,
SCALA_2_12_17, SCALA_2_12_18, SCALA_2_13_5, SCALA_2_13_8, SCALA_2_13_16,
SCALA_2_13_17)
val DEFAULT_SCALA_VERSION = SCALA_2_12_18
@@ -523,6 +524,7 @@ object Utils {
case Some("spark-3.4") => Some(Spark34)
case Some("spark-3.5") => Some(Spark35)
case Some("spark-4.0") => Some(Spark40)
+ case Some("spark-4.1") => Some(Spark41)
case _ => None
}
@@ -991,6 +993,23 @@ object Spark40 extends SparkClientProjects {
override val sparkColumnarShuffleVersion: String = "4"
}
+object Spark41 extends SparkClientProjects {
+
+ val sparkClientProjectPath = "client-spark/spark-3"
+ val sparkClientProjectName = "celeborn-client-spark-4"
+ val sparkClientShadedProjectPath = "client-spark/spark-4-shaded"
+ val sparkClientShadedProjectName = "celeborn-client-spark-4-shaded"
+
+ val lz4JavaVersion = "1.8.0"
+ val sparkProjectScalaVersion = "2.13.17"
+
+ val sparkVersion = "4.1.1"
+ val zstdJniVersion = "1.5.7-6"
+ val scalaBinaryVersion = "2.13"
+
+ override val sparkColumnarShuffleVersion: String = "4"
+}
+
trait SparkClientProjects {
val sparkClientProjectPath: String
diff --git a/tests/spark-it/pom.xml b/tests/spark-it/pom.xml
index a87594f5c..550fa7cb0 100644
--- a/tests/spark-it/pom.xml
+++ b/tests/spark-it/pom.xml
@@ -333,5 +333,23 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>spark-4.1</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+
<artifactId>celeborn-client-spark-3_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+
<artifactId>celeborn-client-spark-3_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
