This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
commit 3fb6d5b829c421c9660439f0808b010e27aabe2c
Author: SteNicholas <programg...@163.com>
AuthorDate: Thu May 29 15:55:09 2025 +0800
[CELEBORN-1413][FOLLOWUP] Support dependencies of spark-4.0 profile
### What changes were proposed in this pull request?
Support dependencies of `spark-4.0` profile.
Follow up #3282.
### Why are the changes needed?
#3282 is lack of dependencies support of `spark-4.0` profile.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Dependencies check: maven-jdk17 (spark-4.0).
Closes #3298 from SteNicholas/CELEBORN-1413.
Authored-by: SteNicholas <programg...@163.com>
Signed-off-by: SteNicholas <programg...@163.com>
---
.github/workflows/deps.yml | 20 +++++++++
dev/dependencies.sh | 4 ++
dev/deps/dependencies-client-spark-4.0 | 80 ++++++++++++++++++++++++++++++++++
3 files changed, 104 insertions(+)
diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml
index d1af2e755..f000eca64 100644
--- a/.github/workflows/deps.yml
+++ b/.github/workflows/deps.yml
@@ -119,3 +119,23 @@ jobs:
- name: Maven Check dependency list
run: |
./dev/dependencies.sh --module ${{ matrix.module }} --check
+
+ maven-jdk17:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ module:
+ - 'spark-4.0'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup JDK 17
+ uses: actions/setup-java@v4
+ with:
+ distribution: zulu
+ java-version: 17
+ cache: maven
+ check-latest: false
+ - name: Maven Check dependency list
+ run: |
+ ./dev/dependencies.sh --module ${{ matrix.module }} --check
diff --git a/dev/dependencies.sh b/dev/dependencies.sh
index e7fbe6463..9b21edec1 100755
--- a/dev/dependencies.sh
+++ b/dev/dependencies.sh
@@ -172,6 +172,10 @@ case "$MODULE" in
MVN_MODULES="client-spark/spark-3"
SBT_PROJECT="celeborn-client-spark-3"
;;
+ "spark-4"*) # Match all versions starting with "spark-4"
+ MVN_MODULES="client-spark/spark-3"
+ SBT_PROJECT="celeborn-client-spark-4"
+ ;;
"flink-1.16")
MVN_MODULES="client-flink/flink-1.16"
SBT_PROJECT="celeborn-client-flink-1_16"
diff --git a/dev/deps/dependencies-client-spark-4.0
b/dev/deps/dependencies-client-spark-4.0
new file mode 100644
index 000000000..3f0cf7f22
--- /dev/null
+++ b/dev/deps/dependencies-client-spark-4.0
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar
+commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-io/2.17.0//commons-io-2.17.0.jar
+commons-lang3/3.17.0//commons-lang3-3.17.0.jar
+commons-logging/1.1.3//commons-logging-1.1.3.jar
+failureaccess/1.0.2//failureaccess-1.0.2.jar
+guava/33.1.0-jre//guava-33.1.0-jre.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
+jackson-annotations/2.15.3//jackson-annotations-2.15.3.jar
+jackson-core/2.15.3//jackson-core-2.15.3.jar
+jackson-databind/2.15.3//jackson-databind-2.15.3.jar
+jackson-module-scala_2.13/2.15.3//jackson-module-scala_2.13-2.15.3.jar
+jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
+jsr305/1.3.9//jsr305-1.3.9.jar
+jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
+leveldbjni-all/1.8//leveldbjni-all-1.8.jar
+lz4-java/1.8.0//lz4-java-1.8.0.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
+netty-all/4.1.118.Final//netty-all-4.1.118.Final.jar
+netty-buffer/4.1.118.Final//netty-buffer-4.1.118.Final.jar
+netty-codec-dns/4.1.118.Final//netty-codec-dns-4.1.118.Final.jar
+netty-codec-haproxy/4.1.118.Final//netty-codec-haproxy-4.1.118.Final.jar
+netty-codec-http/4.1.118.Final//netty-codec-http-4.1.118.Final.jar
+netty-codec-http2/4.1.118.Final//netty-codec-http2-4.1.118.Final.jar
+netty-codec-memcache/4.1.118.Final//netty-codec-memcache-4.1.118.Final.jar
+netty-codec-mqtt/4.1.118.Final//netty-codec-mqtt-4.1.118.Final.jar
+netty-codec-redis/4.1.118.Final//netty-codec-redis-4.1.118.Final.jar
+netty-codec-smtp/4.1.118.Final//netty-codec-smtp-4.1.118.Final.jar
+netty-codec-socks/4.1.118.Final//netty-codec-socks-4.1.118.Final.jar
+netty-codec-stomp/4.1.118.Final//netty-codec-stomp-4.1.118.Final.jar
+netty-codec-xml/4.1.118.Final//netty-codec-xml-4.1.118.Final.jar
+netty-codec/4.1.118.Final//netty-codec-4.1.118.Final.jar
+netty-common/4.1.118.Final//netty-common-4.1.118.Final.jar
+netty-handler-proxy/4.1.118.Final//netty-handler-proxy-4.1.118.Final.jar
+netty-handler/4.1.118.Final//netty-handler-4.1.118.Final.jar
+netty-resolver-dns-classes-macos/4.1.118.Final//netty-resolver-dns-classes-macos-4.1.118.Final.jar
+netty-resolver-dns-native-macos/4.1.118.Final/osx-aarch_64/netty-resolver-dns-native-macos-4.1.118.Final-osx-aarch_64.jar
+netty-resolver-dns-native-macos/4.1.118.Final/osx-x86_64/netty-resolver-dns-native-macos-4.1.118.Final-osx-x86_64.jar
+netty-resolver-dns/4.1.118.Final//netty-resolver-dns-4.1.118.Final.jar
+netty-resolver/4.1.118.Final//netty-resolver-4.1.118.Final.jar
+netty-transport-classes-epoll/4.1.118.Final//netty-transport-classes-epoll-4.1.118.Final.jar
+netty-transport-classes-kqueue/4.1.118.Final//netty-transport-classes-kqueue-4.1.118.Final.jar
+netty-transport-native-epoll/4.1.118.Final/linux-aarch_64/netty-transport-native-epoll-4.1.118.Final-linux-aarch_64.jar
+netty-transport-native-epoll/4.1.118.Final/linux-riscv64/netty-transport-native-epoll-4.1.118.Final-linux-riscv64.jar
+netty-transport-native-epoll/4.1.118.Final/linux-x86_64/netty-transport-native-epoll-4.1.118.Final-linux-x86_64.jar
+netty-transport-native-kqueue/4.1.118.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.118.Final-osx-aarch_64.jar
+netty-transport-native-kqueue/4.1.118.Final/osx-x86_64/netty-transport-native-kqueue-4.1.118.Final-osx-x86_64.jar
+netty-transport-native-unix-common/4.1.118.Final//netty-transport-native-unix-common-4.1.118.Final.jar
+netty-transport-rxtx/4.1.118.Final//netty-transport-rxtx-4.1.118.Final.jar
+netty-transport-sctp/4.1.118.Final//netty-transport-sctp-4.1.118.Final.jar
+netty-transport-udt/4.1.118.Final//netty-transport-udt-4.1.118.Final.jar
+netty-transport/4.1.118.Final//netty-transport-4.1.118.Final.jar
+paranamer/2.8//paranamer-2.8.jar
+protobuf-java/3.25.5//protobuf-java-3.25.5.jar
+scala-library/2.13.16//scala-library-2.13.16.jar
+scala-reflect/2.13.16//scala-reflect-2.13.16.jar
+slf4j-api/1.7.36//slf4j-api-1.7.36.jar
+snakeyaml/2.2//snakeyaml-2.2.jar
+snappy-java/1.1.10.5//snappy-java-1.1.10.5.jar
+zstd-jni/1.5.6-9//zstd-jni-1.5.6-9.jar
