This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ee57ff86f4 [VL] Move Spark 3.5 SMJ-enabled full test jobs into nighly
job (#12529)
ee57ff86f4 is described below
commit ee57ff86f4ffb18e23774cdaf79cc2c39da0503d
Author: JiaKe <[email protected]>
AuthorDate: Mon Sep 21 13:27:28 2026 +0100
[VL] Move Spark 3.5 SMJ-enabled full test jobs into nighly job (#12529)
---
.github/workflows/velox_nightly.yml | 98 +++++++++++++++++++++++++++++++++++++
1 file changed, 98 insertions(+)
diff --git a/.github/workflows/velox_nightly.yml
b/.github/workflows/velox_nightly.yml
index 2e45172b92..f55d5cd187 100644
--- a/.github/workflows/velox_nightly.yml
+++ b/.github/workflows/velox_nightly.yml
@@ -26,6 +26,7 @@ on:
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+ MVN_CMD: 'build/mvn -ntp'
CCACHE_DIR: "${{ github.workspace }}/.ccache"
concurrency:
@@ -180,6 +181,103 @@ jobs:
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7
+ spark-test-spark35-smj:
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ needs: build-native-lib-x86
+ runs-on: ubuntu-22.04
+ env:
+ SPARK_TESTING: true
+ container: apache/gluten:centos-8-jdk8
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-${{github.sha}}
+ path: ./cpp/build/
+ - name: Prepare
+ run: |
+ dnf module -y install python39 && \
+ alternatives --set python3 /usr/bin/python3.9 && \
+ pip3 install setuptools==77.0.3 && \
+ pip3 install pyspark==3.5.5 cython && \
+ pip3 install pandas==2.2.3 pyarrow==20.0.0
+ - name: Build and Run unit test for Spark 3.5.5 (other tests)
+ run: |
+ cd $GITHUB_WORKSPACE/
+ export SPARK_SCALA_VERSION=2.12
+ yum install -y java-17-openjdk-devel
+ export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ export PATH=$JAVA_HOME/bin:$PATH
+ java -version
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest
+ - name: Upload test report
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ github.job }}-report
+ path: '**/surefire-reports/TEST-*.xml'
+ - name: Upload unit tests log files
+ if: ${{ !success() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ github.job }}-test-log
+ path: |
+ **/target/*.log
+ **/gluten-ut/**/hs_err_*.log
+ **/gluten-ut/**/core.*
+
+ spark-test-spark35-slow-smj:
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ needs: build-native-lib-x86
+ runs-on: ubuntu-22.04
+ env:
+ SPARK_TESTING: true
+ LANG: C.UTF-8 #TODO remove after image update
+ LC_ALL: C.UTF-8
+ container: apache/gluten:centos-8-jdk8
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-${{github.sha}}
+ path: ./cpp/build/
+ - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image
update
+ run: |
+ rm -rf /opt/shims/spark35
+ bash .github/workflows/util/install-spark-resources.sh 3.5
+ - name: Build and Run unit test for Spark 3.5.5 (slow tests)
+ run: |
+ cd $GITHUB_WORKSPACE/
+ yum install -y java-17-openjdk-devel
+ export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ export PATH=$JAVA_HOME/bin:$PATH
+ java -version
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
+ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
+ -DtagsToInclude=org.apache.spark.tags.SlowHiveTest
+ - name: Upload test report
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ github.job }}-report
+ path: '**/surefire-reports/TEST-*.xml'
+ - name: Upload unit tests log files
+ if: ${{ !success() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ github.job }}-test-log
+ path: |
+ **/target/*.log
+ **/gluten-ut/**/hs_err_*.log
+ **/gluten-ut/**/core.*
+
#build and package for arm64
build-native-lib-centos-8-arm64:
if: ${{ startsWith(github.repository, 'apache/') }}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
