This is an automated email from the ASF dual-hosted git repository.
kerwinzhang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 32808dd22 [CELEBORN] Upgrade celeborn to 0.4.1 to support scala
2.13-based compilation (#6226)
32808dd22 is described below
commit 32808dd22a0384d0e0bb5011bf2393710a4d5942
Author: Kerwin Zhang <[email protected]>
AuthorDate: Thu Jun 27 09:58:25 2024 +0800
[CELEBORN] Upgrade celeborn to 0.4.1 to support scala 2.13-based
compilation (#6226)
---
.github/workflows/velox_docker.yml | 6 +++---
docs/get-started/ClickHouse.md | 12 ++++++------
.../shuffle/gluten/celeborn/CelebornShuffleManager.java | 8 +++++++-
.../spark/shuffle/gluten/celeborn/CelebornUtils.java | 14 ++++++++++++--
pom.xml | 2 +-
tools/gluten-it/pom.xml | 4 ++--
6 files changed, 31 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/velox_docker.yml
b/.github/workflows/velox_docker.yml
index 31796c15b..d110d0a6d 100644
--- a/.github/workflows/velox_docker.yml
+++ b/.github/workflows/velox_docker.yml
@@ -521,7 +521,7 @@ jobs:
fail-fast: false
matrix:
spark: ["spark-3.2"]
- celeborn: ["celeborn-0.4.0", "celeborn-0.3.2"]
+ celeborn: ["celeborn-0.4.1", "celeborn-0.3.2-incubating"]
runs-on: ubuntu-20.04
container: ubuntu:22.04
steps:
@@ -557,8 +557,8 @@ jobs:
fi
echo "EXTRA_PROFILE: ${EXTRA_PROFILE}"
cd /opt && mkdir -p celeborn && \
- wget https://archive.apache.org/dist/incubator/celeborn/${{
matrix.celeborn }}-incubating/apache-${{ matrix.celeborn }}-incubating-bin.tgz
&& \
- tar xzf apache-${{ matrix.celeborn }}-incubating-bin.tgz -C
/opt/celeborn --strip-components=1 && cd celeborn && \
+ wget https://archive.apache.org/dist/celeborn/${{ matrix.celeborn
}}/apache-${{ matrix.celeborn }}-bin.tgz && \
+ tar xzf apache-${{ matrix.celeborn }}-bin.tgz -C /opt/celeborn
--strip-components=1 && cd celeborn && \
mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \
bash -c "echo -e
'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g'
> ./conf/celeborn-env.sh" && \
bash -c "echo -e 'celeborn.worker.commitFiles.threads
128\nceleborn.worker.sortPartition.threads 64' > ./conf/celeborn-defaults.conf"
&& \
diff --git a/docs/get-started/ClickHouse.md b/docs/get-started/ClickHouse.md
index 4352a99e5..ab24de7a4 100644
--- a/docs/get-started/ClickHouse.md
+++ b/docs/get-started/ClickHouse.md
@@ -679,13 +679,13 @@
spark.shuffle.manager=org.apache.spark.shuffle.gluten.celeborn.CelebornShuffleMa
quickly start a celeborn cluster
```shell
-wget
https://archive.apache.org/dist/incubator/celeborn/celeborn-0.3.0-incubating/apache-celeborn-0.3.0-incubating-bin.tgz
&& \
-tar -zxvf apache-celeborn-0.3.0-incubating-bin.tgz && \
-mv apache-celeborn-0.3.0-incubating-bin/conf/celeborn-defaults.conf.template
apache-celeborn-0.3.0-incubating-bin/conf/celeborn-defaults.conf && \
-mv apache-celeborn-0.3.0-incubating-bin/conf/log4j2.xml.template
apache-celeborn-0.3.0-incubating-bin/conf/log4j2.xml && \
+wget
https://archive.apache.org/dist/celeborn/celeborn-0.3.2-incubating/apache-celeborn-0.3.2-incubating-bin.tgz
&& \
+tar -zxvf apache-celeborn-0.3.2-incubating-bin.tgz && \
+mv apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf.template
apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf && \
+mv apache-celeborn-0.3.2-incubating-bin/conf/log4j2.xml.template
apache-celeborn-0.3.2-incubating-bin/conf/log4j2.xml && \
mkdir /opt/hadoop && chmod 777 /opt/hadoop && \
-echo -e "celeborn.worker.flusher.threads 4\nceleborn.worker.storage.dirs
/tmp\nceleborn.worker.monitor.disk.enabled false" >
apache-celeborn-0.3.0-incubating-bin/conf/celeborn-defaults.conf && \
-bash apache-celeborn-0.3.0-incubating-bin/sbin/start-master.sh && bash
apache-celeborn-0.3.0-incubating-bin/sbin/start-worker.sh
+echo -e "celeborn.worker.flusher.threads 4\nceleborn.worker.storage.dirs
/tmp\nceleborn.worker.monitor.disk.enabled false" >
apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf && \
+bash apache-celeborn-0.3.2-incubating-bin/sbin/start-master.sh && bash
apache-celeborn-0.3.2-incubating-bin/sbin/start-worker.sh
```
### Columnar shuffle mode
diff --git
a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
index f454cf00c..d196691d1 100644
---
a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
+++
b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
@@ -217,7 +217,13 @@ public class CelebornShuffleManager implements
ShuffleManager {
}
}
return CelebornUtils.unregisterShuffle(
- lifecycleManager, shuffleClient, shuffleIdTracker, shuffleId,
appUniqueId, isDriver());
+ lifecycleManager,
+ shuffleClient,
+ shuffleIdTracker,
+ shuffleId,
+ appUniqueId,
+ throwsFetchFailure,
+ isDriver());
}
@Override
diff --git
a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java
b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java
index 9dd4e1d11..6b4229ad3 100644
---
a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java
+++
b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java
@@ -49,11 +49,21 @@ public class CelebornUtils {
Object shuffleIdTracker,
int appShuffleId,
String appUniqueId,
+ boolean throwsFetchFailure,
boolean isDriver) {
try {
- // for Celeborn 0.4.0
try {
- if (lifecycleManager != null) {
+ try {
+ // for Celeborn 0.4.1
+ if (lifecycleManager != null) {
+ Method unregisterAppShuffle =
+ lifecycleManager
+ .getClass()
+ .getMethod("unregisterAppShuffle", int.class,
boolean.class);
+ unregisterAppShuffle.invoke(lifecycleManager, appShuffleId,
throwsFetchFailure);
+ }
+ } catch (NoSuchMethodException ex) {
+ // for Celeborn 0.4.0
Method unregisterAppShuffle =
lifecycleManager.getClass().getMethod("unregisterAppShuffle",
int.class);
unregisterAppShuffle.invoke(lifecycleManager, appShuffleId);
diff --git a/pom.xml b/pom.xml
index 81ce0e5d4..887839ce5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -53,7 +53,7 @@
<delta.package.name>delta-core</delta.package.name>
<delta.version>2.4.0</delta.version>
<delta.binary.version>24</delta.binary.version>
- <celeborn.version>0.3.2-incubating</celeborn.version>
+ <celeborn.version>0.4.1</celeborn.version>
<uniffle.version>0.8.0</uniffle.version>
<arrow.version>15.0.0</arrow.version>
<arrow-gluten.version>15.0.0-gluten</arrow-gluten.version>
diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
index 3f1760069..71db637a8 100644
--- a/tools/gluten-it/pom.xml
+++ b/tools/gluten-it/pom.xml
@@ -21,7 +21,7 @@
<spark.version>3.4.2</spark.version>
<scala.binary.version>2.12</scala.binary.version>
<spark.major.version>3</spark.major.version>
- <celeborn.version>0.3.0-incubating</celeborn.version>
+ <celeborn.version>0.3.2-incubating</celeborn.version>
<uniffle.version>0.8.0</uniffle.version>
<gluten.version>1.2.0-SNAPSHOT</gluten.version>
<guava.version>32.0.1-jre</guava.version>
@@ -167,7 +167,7 @@
<profile>
<id>celeborn-0.4</id>
<properties>
- <celeborn.version>0.4.0-incubating</celeborn.version>
+ <celeborn.version>0.4.1</celeborn.version>
</properties>
</profile>
</profiles>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
