This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 8ace427b4e [GLUTEN-9860][VL] Add nightly packages for ARM (#10204)
8ace427b4e is described below
commit 8ace427b4eff42525c90a4d7c3ecde71ec9fc1ec
Author: Yuan <[email protected]>
AuthorDate: Wed Jul 23 10:58:50 2025 +0100
[GLUTEN-9860][VL] Add nightly packages for ARM (#10204)
This patch adds nightly builds for ARM and will sync to apache nightly site
for Gluten
Signed-off-by: Yuan <[email protected]>
---
.github/workflows/velox_nightly.yml | 165 +++++++++++++++++++++++++++++++++++-
1 file changed, 164 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/velox_nightly.yml
b/.github/workflows/velox_nightly.yml
index 17764e5b8b..5205f643c1 100644
--- a/.github/workflows/velox_nightly.yml
+++ b/.github/workflows/velox_nightly.yml
@@ -117,7 +117,6 @@ jobs:
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
-
build-bundle-package-centos8-jdk17:
needs: build-native-lib
runs-on: ubuntu-22.04
@@ -166,3 +165,167 @@ jobs:
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
+
+
+ #TODO: split ARM section into different file
+ build-native-lib-centos-8-arm64:
+ runs-on: ubuntu-24.04-arm
+ container: apache/gluten:vcpkg-centos-8
+ steps:
+ - uses: actions/checkout@v4
+ - name: Get Ccache
+ uses: actions/cache/restore@v4
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-centos8-release-default-${{runner.arch}}-${{github.sha}}
+ restore-keys: |
+ ccache-centos8-release-default-${{runner.arch}}
+ - name: Build Gluten native libraries
+ run: |
+ df -a
+ bash dev/ci-velox-buildstatic-centos-8.sh
+ ccache -s
+ mkdir -p $GITHUB_WORKSPACE/.m2/repository/org/apache/arrow/
+ cp -r /root/.m2/repository/org/apache/arrow/*
$GITHUB_WORKSPACE/.m2/repository/org/apache/arrow/
+ - name: "Save ccache"
+ uses: actions/cache/save@v4
+ id: ccache
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-centos8-release-default-${{runner.arch}}-${{github.sha}}
+ - uses: actions/upload-artifact@v4
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases/
+ if-no-files-found: error
+ - uses: actions/upload-artifact@v4
+ with:
+ name: velox-arrow-jar-centos-8-${{github.sha}}
+ path: .m2/repository/org/apache/arrow/
+ if-no-files-found: error
+
+ build-bundle-package-centos8-jdk8-arm64:
+ needs: build-native-lib-centos-8-arm64
+ runs-on: ubuntu-22.04-arm
+ container: centos:8
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download All Arrow Jar Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-arrow-jar-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
+ - name: Setup java and maven
+ run: |
+ sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
+ sed -i
's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g'
/etc/yum.repos.d/CentOS-* && \
+ yum update -y && yum install -y java-1.8.0-openjdk-devel wget
+ $SETUP install_maven
+ - name: Get current date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+ - name: Build package for Spark
+ run: |
+ cd $GITHUB_WORKSPACE/ && \
+ export MAVEN_HOME=/usr/lib/maven && \
+ export PATH=${PATH}:${MAVEN_HOME}/bin && \
+ mvn clean install -Pspark-3.2 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
+ mvn clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
+ mvn clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
+ mvn clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Puniffle
-DskipTests -Dmaven.source.skip
+ - name: Upload bundle package
+ uses: actions/upload-artifact@v4
+ with:
+ name: nightly-gluten-velox-bundle-package-jdk8-arm64-${{
steps.date.outputs.date }}
+ path: package/target/gluten-velox-bundle-*.jar
+ retention-days: 7
+
+ build-bundle-package-centos8-jdk17-arm64:
+ needs: build-native-lib-centos-8-arm64
+ runs-on: ubuntu-22.04-arm
+ container: centos:8
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-centos-8-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Download All Arrow Jar Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-arrow-jar-centos-8-${{github.sha}}
+ path: /root/.m2/repository/org/apache/arrow/
+ - name: Setup java and maven
+ run: |
+ sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
+ sed -i
's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g'
/etc/yum.repos.d/CentOS-* && \
+ yum update -y && yum install -y java-17-openjdk-devel wget
+ $SETUP install_maven
+ - name: Get current date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+ - name: Build package for Spark
+ run: |
+ cd $GITHUB_WORKSPACE/ && \
+ export MAVEN_HOME=/usr/lib/maven && \
+ export PATH=${PATH}:${MAVEN_HOME}/bin && \
+ mvn clean install -Pspark-3.4 -Pjava-17 -Pbackends-velox -Pceleborn
-Puniffle -Piceberg -Phudi -Pdelta -DskipTests -Dmaven.source.skip
+ mvn clean install -Pspark-3.5 -Pjava-17 -Pbackends-velox -Pceleborn
-Puniffle -Piceberg -Phudi -Pdelta -DskipTests -Dmaven.source.skip
+ - name: Upload bundle package
+ uses: actions/upload-artifact@v4
+ with:
+ name: nightly-gluten-velox-bundle-package-jdk17-arm64-${{
steps.date.outputs.date }}
+ path: package/target/gluten-velox-bundle-*.jar
+ retention-days: 7
+
+ upload-jdk8-package-arm64:
+ needs: [build-bundle-package-centos8-jdk8-arm64]
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Get current date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+ - name: Download JDK8 Package Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: nightly-gluten-velox-bundle-package-jdk8-arm64-${{
steps.date.outputs.date }}
+ path: package/
+ - name: rsync to apache nightly
+ uses: burnett01/[email protected]
+ with:
+ switches: -avzr
+ path: package/target/gluten-velox-bundle-*.jar
+ remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH
}}/gluten/nightly-release-jdk8
+ remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
+ remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
+ remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
+ remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
+
+ upload-jdk17-package-arm64:
+ needs: [build-bundle-package-centos8-jdk17-arm64]
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Get current date
+ id: date
+ run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+ - name: Download JDK17 Package Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: nightly-gluten-velox-bundle-package-jdk17-arm64-${{
steps.date.outputs.date }}
+ path: package/
+ - name: rsync to apache nightly
+ uses: burnett01/[email protected]
+ with:
+ switches: -avzr
+ path: package/target/gluten-velox-bundle-*.jar
+ remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH
}}/gluten/nightly-release-jdk17
+ remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
+ remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
+ remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
+ remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
