This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c7de93e1c4 [VL] Refine docker image build to speedup CI jobs(#11835)
c7de93e1c4 is described below
commit c7de93e1c4cd5fa28c30801475368946282184cb
Author: Yuan <[email protected]>
AuthorDate: Thu Jul 30 01:24:50 2026 -0700
[VL] Refine docker image build to speedup CI jobs(#11835)
This patch refines the docker file for gpu image.
switch to use centos9
install cuda-toolkit-13.1
gcc-14 is used
The image size is reduced from ~27GB to ~13GB
The patch also fixed the gcc-12 installation on centos8
The patch also installed spark-41 to CI image to save time on downloading
---------
Signed-off-by: Yuan <[email protected]>
---
dev/docker/Dockerfile.centos8-dynamic-build | 11 +++++++++--
dev/docker/Dockerfile.centos9-dynamic-build | 1 +
dev/docker/cudf/Dockerfile | 21 ++++++++++-----------
3 files changed, 20 insertions(+), 13 deletions(-)
diff --git a/dev/docker/Dockerfile.centos8-dynamic-build
b/dev/docker/Dockerfile.centos8-dynamic-build
index aa3bc39882..59bae2a0cb 100644
--- a/dev/docker/Dockerfile.centos8-dynamic-build
+++ b/dev/docker/Dockerfile.centos8-dynamic-build
@@ -30,7 +30,13 @@ RUN set -ex; \
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* ||
true; \
sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true; \
yum update -y && yum install -y epel-release sudo dnf && yum install -y
ccache; \
- dnf install -y --setopt=install_weak_deps=False gcc-toolset-11; \
+ dnf install -y centos-release-stream; \
+ sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-*
|| true; \
+ sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-Stream-* || true; \
+ dnf swap -y --allowerasing centos-{linux,stream}-repos; \
+ sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-*
|| true; \
+ sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-Stream-* || true; \
+ dnf install -y --setopt=install_weak_deps=False gcc-toolset-12; \
echo "check_certificate = off" >> ~/.wgetrc; \
yum install -y java-${JAVA_VERSION}-openjdk-devel patch wget git perl; \
mirror_host="https://www.apache.org/dyn/closer.lua";; \
@@ -45,12 +51,13 @@ RUN set -ex; \
./install-spark-resources.sh 3.5; \
./install-spark-resources.sh 3.5-scala2.13; \
./install-spark-resources.sh 4.0; \
+ ./install-spark-resources.sh 4.1; \
ARCH=$(uname -m); \
if [[ "$ARCH" == "aarch64" || "$ARCH" == "ppc64le" ]]; then \
export CPU_TARGET="$ARCH"; \
fi; \
cd /opt/gluten; \
- source /opt/rh/gcc-toolset-11/enable; \
+ source /opt/rh/gcc-toolset-12/enable; \
export VELOX_BUILD_SHARED=ON; \
./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow; \
./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
diff --git a/dev/docker/Dockerfile.centos9-dynamic-build
b/dev/docker/Dockerfile.centos9-dynamic-build
index f160164099..0d7246b4a2 100644
--- a/dev/docker/Dockerfile.centos9-dynamic-build
+++ b/dev/docker/Dockerfile.centos9-dynamic-build
@@ -41,6 +41,7 @@ RUN set -ex; \
./install-spark-resources.sh 3.5; \
./install-spark-resources.sh 3.5-scala2.13; \
./install-spark-resources.sh 4.0; \
+ ./install-spark-resources.sh 4.1; \
ARCH=$(uname -m); \
if [[ "$ARCH" == "aarch64" || "$ARCH" == "ppc64le" ]]; then \
export CPU_TARGET="$ARCH"; \
diff --git a/dev/docker/cudf/Dockerfile b/dev/docker/cudf/Dockerfile
index 8f74b41528..071e153cb6 100644
--- a/dev/docker/cudf/Dockerfile
+++ b/dev/docker/cudf/Dockerfile
@@ -15,23 +15,22 @@
# limitations under the License.
#
-FROM ghcr.io/facebookincubator/velox-dev:adapters
-ENV CUDA_ARCHITECTURES=70
+FROM quay.io/centos/centos:stream9
+ENV CUDA_ARCHITECTURES=75
ENV
LD_LIBRARY_PATH=/opt/gluten/ep/build-velox/build/velox_ep/_build/release/_deps/curl-build/lib:$LD_LIBRARY_PATH
+ENV CC=/opt/rh/gcc-toolset-14/root/bin/gcc \
+ CXX=/opt/rh/gcc-toolset-14/root/bin/g++
-
-RUN yum install -y sudo patch perl && \
- dnf remove -y cuda-toolkit-12* && dnf install -y cuda-toolkit-13-1; \
+RUN dnf config-manager --add-repo
"https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo";;
\
+ dnf update; \
+ dnf install -y sudo patch maven perl git gcc-toolset-14 cuda-toolkit-13-1
&& \
dnf autoremove -y && dnf clean all; \
- rm -rf /opt/rh/gcc-toolset-12 && ln -s /opt/rh/gcc-toolset-14
/opt/rh/gcc-toolset-12; \
- ln -sf /usr/local/bin/cmake /usr/bin && \
- git clone --depth=1 https://github.com/apache/gluten /opt/gluten && \
+ git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten
&& \
cd /opt/gluten && \
source /opt/rh/gcc-toolset-14/enable && \
- bash ./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=ON
--spark_version=3.4 --build_tests=ON --build_benchmarks=ON --enable_gpu=ON && \
+ bash ./dev/buildbundle-veloxbe.sh --run_setup_script=ON --build_arrow=ON
--spark_version=3.5 --build_tests=ON --build_benchmarks=ON --enable_gpu=ON && \
rm -rf /opt/gluten && \
- rm -rf /root/.cache/ccache
-
+ rm -rf /root/.cache/ccache
# You can try the data in folder
backends-velox/src/test/resources/tpch-data-parquet
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
