This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 82b70aab98 [INFRA] Use hadoop 2.10.2 for uniffle test (#12111)
82b70aab98 is described below
commit 82b70aab9857e3991f2dfdb5f9834eb83a8884d5
Author: Wechar Yu <[email protected]>
AuthorDate: Fri May 29 00:35:23 2026 +0800
[INFRA] Use hadoop 2.10.2 for uniffle test (#12111)
* [INFRA] Use hadoop 2.10.2 for uniffle test
* fix CI
* remove unused uniffle tarball in centos9 Dockerfile
---
.github/workflows/velox_backend_x86.yml | 6 +++---
dev/docker/Dockerfile.centos8-dynamic-build | 2 +-
dev/docker/Dockerfile.centos9-dynamic-build | 2 --
3 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 7342c62a15..8672e5ecc4 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -554,7 +554,7 @@ jobs:
matrix:
spark: [ "spark-3.3" ]
uniffle: [ "0.10.0" ]
- hadoop: [ "2.8.5" ]
+ hadoop: [ "2.10.2" ]
runs-on: ubuntu-22.04
container: apache/gluten:centos-8-jdk8
steps:
@@ -573,10 +573,10 @@ jobs:
run: |
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
if [ ! -e "/opt/apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz" ];
then
- ${WGET_CMD} https://archive.apache.org/dist/uniffle/${{
matrix.uniffle }}/apache-uniffle-${{ matrix.uniffle }}-bin.tar.gz -P /opt
+ ${WGET_CMD} https://www.apache.org/dyn/closer.lua/uniffle/${{
matrix.uniffle }}/apache-uniffle-${{ matrix.uniffle
}}-bin.tar.gz?action=download -O /opt/apache-uniffle-${{ matrix.uniffle
}}-bin.tar.gz
fi
if [ ! -e "/opt/hadoop-${{ matrix.hadoop }}.tar.gz" ]; then
- ${WGET_CMD}
https://archive.apache.org/dist/hadoop/common/hadoop-${{ matrix.hadoop
}}/hadoop-${{ matrix.hadoop }}.tar.gz -P /opt
+ ${WGET_CMD}
https://www.apache.org/dyn/closer.lua/hadoop/common/hadoop-${{ matrix.hadoop
}}/hadoop-${{ matrix.hadoop }}.tar.gz?action=download -O /opt/hadoop-${{
matrix.hadoop }}.tar.gz
fi
cd /opt && rm -rf shims && \
mkdir /opt/uniffle && tar xzf apache-uniffle-${{ matrix.uniffle
}}-bin.tar.gz -C /opt/uniffle --strip-components=1 && \
diff --git a/dev/docker/Dockerfile.centos8-dynamic-build
b/dev/docker/Dockerfile.centos8-dynamic-build
index 90435d0412..ff1fb48a86 100644
--- a/dev/docker/Dockerfile.centos8-dynamic-build
+++ b/dev/docker/Dockerfile.centos8-dynamic-build
@@ -37,7 +37,7 @@ RUN set -ex; \
wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-O /opt/apache-celeborn-0.5.4-bin.tgz; \
wget -nv
${mirror_host}/celeborn/celeborn-0.6.3/apache-celeborn-0.6.3-bin.tgz?action=download
-O /opt/apache-celeborn-0.6.3-bin.tgz; \
wget -nv
${mirror_host}/uniffle/0.10.0/apache-uniffle-0.10.0-bin.tar.gz?action=download
-O /opt/apache-uniffle-0.10.0-bin.tar.gz; \
- wget -nv
${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz?action=download
-O /opt/hadoop-2.8.5.tar.gz; \
+ wget -nv
${mirror_host}/hadoop/common/hadoop-2.10.2/hadoop-2.10.2.tar.gz?action=download
-O /opt/hadoop-2.10.2.tar.gz; \
git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \
cd /opt/gluten/.github/workflows/util/; \
./install-spark-resources.sh 3.3; \
diff --git a/dev/docker/Dockerfile.centos9-dynamic-build
b/dev/docker/Dockerfile.centos9-dynamic-build
index 980737df7b..4c490d3477 100644
--- a/dev/docker/Dockerfile.centos9-dynamic-build
+++ b/dev/docker/Dockerfile.centos9-dynamic-build
@@ -34,8 +34,6 @@ RUN set -ex; \
mirror_host="https://www.apache.org/dyn/closer.lua";; \
wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-O /opt/apache-celeborn-0.5.4-bin.tgz; \
wget -nv
${mirror_host}/celeborn/celeborn-0.6.3/apache-celeborn-0.6.3-bin.tgz?action=download
-O /opt/apache-celeborn-0.6.3-bin.tgz; \
- wget -nv
${mirror_host}/uniffle/0.10.0/apache-uniffle-0.10.0-bin.tar.gz?action=download
-O /opt/apache-uniffle-0.10.0-bin.tar.gz; \
- wget -nv
${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz?action=download
-O /opt/hadoop-2.8.5.tar.gz; \
git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \
cd /opt/gluten/.github/workflows/util/; \
./install-spark-resources.sh 3.3; \
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
