This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 8bd4e2b783 [VL][CI] Adding docker image for maven cache (#11655)
8bd4e2b783 is described below
commit 8bd4e2b783aa6c4a3030e929e8a5b5f05ab7fb87
Author: Shahad Shamsan <[email protected]>
AuthorDate: Thu Apr 2 17:39:21 2026 +0300
[VL][CI] Adding docker image for maven cache (#11655)
This patch added a docker image with maven cache for spark package/test,
covering 3.3/3.4/3.5/4.0/4.1.
A second patch will be made to enable this cache in the CI jobs
---
.github/workflows/docker_image.yml | 26 +++++++++
dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache | 74 ++++++++++++++++++++++++++
2 files changed, 100 insertions(+)
diff --git a/.github/workflows/docker_image.yml
b/.github/workflows/docker_image.yml
index 3996b5b12c..b1e9a9cc51 100644
--- a/.github/workflows/docker_image.yml
+++ b/.github/workflows/docker_image.yml
@@ -24,6 +24,7 @@ on:
- '.github/workflows/util/install-resources.sh'
- 'dev/docker/*'
- 'dev/docker/cudf/*'
+ - 'dev/docker/ubuntu/*'
schedule:
- cron: '0 20 * * 0'
@@ -112,6 +113,31 @@ jobs:
push: true
tags: ${{ env.DOCKERHUB_REPO }}:centos-9-jdk8-cudf
+ build-ubuntu22-m2-cache:
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ runs-on: ubuntu-22.04
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Login to Docker Hub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USER }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Build and push Docker image
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ file: dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache
+ push: true
+ tags: ${{ env.DOCKERHUB_REPO }}:ubuntu22-m2-cache
+
build-centos-9-jdk17-cuda12_9-cudf:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
diff --git a/dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache
b/dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache
new file mode 100644
index 0000000000..119f22fe5b
--- /dev/null
+++ b/dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-arm64
+ENV PATH=$JAVA_HOME/bin:$PATH
+
+RUN apt-get update && apt-get install -y \
+ sudo git build-essential cmake ninja-build \
+ python3 curl ccache wget unzip \
+ openjdk-17-jdk maven \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN git clone --depth=1 https://github.com/apache/incubator-gluten.git
/opt/gluten
+
+WORKDIR /opt/gluten
+
+# Build Velox dependencies with Arrow enabled
+RUN bash ./dev/builddeps-veloxbe.sh \
+ --run_setup_script=ON \
+ build_arrow
+
+# Download Maven dependencies for common Spark versions
+# This pre-populates the Maven cache with all necessary dependencies
+RUN set -ex; \
+ export MVN_CMD="build/mvn -ntp"; \
+
+ # Spark 4.1
+ $MVN_CMD
-Pjava-17,scala-2.13,spark-4.1,backends-velox,hadoop-3.3,spark-ut,delta \
+ dependency:go-offline -DskipTests || true; \
+
+ # Spark 4.0
+ $MVN_CMD -Pjava-17,scala-2.13,spark-4,backends-velox,hadoop-3.3,spark-ut \
+ -Piceberg,iceberg-test,delta,paimon \
+ dependency:go-offline -DskipTests || true; \
+
+ # Spark 3.5
+ $MVN_CMD -Pjava-17,spark-3.5,backends-velox,hadoop-3.3,spark-ut \
+ -Piceberg,iceberg-test,delta,paimon \
+ dependency:go-offline -DskipTests || true; \
+
+ # Spark 3.4
+ $MVN_CMD -Pjava-17,spark-3.4,backends-velox,hadoop-3.3,spark-ut \
+ -Piceberg,iceberg-test,delta,paimon \
+ dependency:go-offline -DskipTests || true; \
+
+ # Spark 3.3
+ $MVN_CMD -Pjava-17,spark-3.3,backends-velox,hadoop-3.3,spark-ut \
+ -Piceberg,delta,paimon \
+ dependency:go-offline -DskipTests || true
+
+WORKDIR /work
+
+# Clean up build artifacts but keep m2
+RUN rm -rf /opt/gluten/cpp/build \
+ && rm -rf /opt/gluten/ep/build-velox/build \
+ && rm -rf /root/.cache/ccache
+
+# Default command
+CMD ["/bin/bash"]
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
