This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 664ef23270 [VL] Use clean docker image for Celeborn tests (#11250)
664ef23270 is described below
commit 664ef2327051a26d9d05aad8ff97341f5abc829c
Author: Yuan <[email protected]>
AuthorDate: Fri Dec 5 01:16:28 2025 +0000
[VL] Use clean docker image for Celeborn tests (#11250)
* use clean docker image for celeborn
Signed-off-by: Yuan <[email protected]>
* speedup download with apache mirror
Signed-off-by: Yuan <[email protected]>
---------
Signed-off-by: Yuan <[email protected]>
---
.github/workflows/velox_backend_x86.yml | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 420763dd1b..d2a9768dde 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -561,7 +561,7 @@ jobs:
celeborn: [ "celeborn-0.6.1", "celeborn-0.5.4"]
writer: ["sort", "hash"]
runs-on: ubuntu-22.04
- container: apache/gluten:centos-8-jdk8
+ container: centos:8
steps:
- uses: actions/checkout@v2
- name: Download All Native Artifacts
@@ -574,6 +574,17 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
+ - name: Update mirror list
+ run: |
+ sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
+ sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
+ - name: Setup java and maven
+ run: |
+ yum update -y && yum install -y java-1.8.0-openjdk-devel wget
+ $SETUP install_maven
+ - name: Set environment variables
+ run: |
+ echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
- name: Build for Spark ${{ matrix.spark }}
run: |
cd $GITHUB_WORKSPACE/
@@ -589,7 +600,7 @@ jobs:
echo "EXTRA_PROFILE: ${EXTRA_PROFILE}"
if [ ! -e "/opt/apache-${{ matrix.celeborn }}-bin.tgz" ]; then
echo "WARNING: please pre-install your required package in docker
image since the downloading is throttled by this site."
- wget -nv https://archive.apache.org/dist/celeborn/${{
matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz -P /opt/
+ wget -nv https://www.apache.org/dyn/closer.lua/celeborn/${{
matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz?action=download -O
/opt/apache-${{ matrix.celeborn }}-bin.tgz
fi
cd /opt && rm -rf shims && mkdir -p celeborn && \
tar xzf apache-${{ matrix.celeborn }}-bin.tgz -C /opt/celeborn
--strip-components=1 && cd celeborn && \
@@ -610,7 +621,7 @@ jobs:
--extra-conf=spark.celeborn.client.eagerlyCreateInputStream.threads=4 \
--benchmark-type=ds --error-on-memleak \
--off-heap-size=16g -s=1.0 --threads=16 --iterations=1
- if false; then # TODO: fix celeborn rss sort issue
+ if [ "${{ matrix.writer }}" = "sort" ]; then
GLUTEN_IT_JVM_ARGS=-Xmx16G sbin/gluten-it.sh queries-compare \
--local --preset=velox-with-celeborn
--extra-conf=spark.celeborn.client.spark.shuffle.writer=${{ matrix.writer }} \
--extra-conf=spark.gluten.sql.columnar.shuffle.celeborn.useRssSort=false \
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
