This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch feature/ansi-ci-baseline in repository https://gitbox.apache.org/repos/asf/gluten.git
commit fa14e565f3cf360203eafeaae15023e9fe212725 Author: baibaichen <[email protected]> AuthorDate: Fri Mar 27 13:38:11 2026 +0800 ci: split ANSI test into parallel backends-velox and spark-ut jobs Split single spark-test-spark41-ansi job into two parallel jobs: - spark-test-backends-velox-ansi: runs only backends-velox tests (no -Pspark-ut profile) - spark-test-spark-ut-ansi: runs spark-ut tests using wildcardSuites to filter scalatest suites under org.apache.spark.* This fixes spark-ut being skipped when backends-velox tests fail (Maven -fae doesn't work for parent-child module dependencies). Also adds set -o pipefail to propagate Maven exit code through tee. Co-Authored-By: Claude Opus 4.6 <[email protected]> --- .github/workflows/velox_backend_ansi.yml | 102 +++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 12 deletions(-) diff --git a/.github/workflows/velox_backend_ansi.yml b/.github/workflows/velox_backend_ansi.yml index fdd7b79f8a..51af6edb31 100644 --- a/.github/workflows/velox_backend_ansi.yml +++ b/.github/workflows/velox_backend_ansi.yml @@ -88,7 +88,7 @@ jobs: path: .m2/repository/org/apache/arrow/ if-no-files-found: error - spark-test-spark41-ansi: + spark-test-backends-velox-ansi: needs: build-native-lib runs-on: ubuntu-22.04 env: @@ -119,8 +119,9 @@ jobs: rm -rf /opt/shims/spark41 bash .github/workflows/util/install-resources.sh 4.1 mv /opt/shims/spark41/spark_home/assembly/target/scala-2.12 /opt/shims/spark41/spark_home/assembly/target/scala-2.13 - - name: "Spark 4.1 Unit Tests (ANSI ON)" + - name: "Spark 4.1 backends-velox Tests (ANSI ON)" run: | + set -o pipefail cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 yum install -y java-17-openjdk-devel @@ -128,38 +129,115 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE" java -version - $MVN_CMD clean test -fae -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ - -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \ + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ - 2>&1 | tee ansi-test-output.log + 2>&1 | tee backends-velox-test-output.log - name: "Parse test results" if: always() run: | echo "=========================================" - echo " Test Results (Spark 4.1, ANSI=$SPARK_ANSI_SQL_MODE)" + echo " backends-velox (Spark 4.1, ANSI=$SPARK_ANSI_SQL_MODE)" echo "=========================================" echo "" echo "--- Test Summary ---" - grep -E "Tests run:.*Failures:|BUILD " ansi-test-output.log || echo "(no summary found)" + grep -E "Tests run:.*Failures:|BUILD " backends-velox-test-output.log || echo "(no summary found)" echo "" echo "--- Failed Tests ---" - grep -B1 "<<< FAIL!" ansi-test-output.log || echo "(no failures)" + grep -B1 "<<< FAIL!" backends-velox-test-output.log || echo "(no failures)" echo "" echo "--- Error Tests ---" - grep -B1 "<<< ERROR!" ansi-test-output.log || echo "(no errors)" + grep -B1 "<<< ERROR!" backends-velox-test-output.log || echo "(no errors)" - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: spark-test-spark41-ansi-report + name: spark-test-backends-velox-ansi-report path: | **/surefire-reports/TEST-*.xml - ansi-test-output.log + backends-velox-test-output.log - name: Upload log files if: always() uses: actions/upload-artifact@v4 with: - name: spark-test-spark41-ansi-logs + name: spark-test-backends-velox-ansi-logs + path: | + **/target/*.log + + spark-test-spark-ut-ansi: + needs: build-native-lib + runs-on: ubuntu-22.04 + env: + SPARK_TESTING: true + container: apache/gluten:centos-9-jdk17 + steps: + - uses: actions/checkout@v4 + - name: Download Native Lib + uses: actions/download-artifact@v4 + with: + name: velox-native-lib-ansi-${{github.sha}} + path: ./cpp/build/ + - name: Download Arrow Jars + uses: actions/download-artifact@v4 + with: + name: arrow-jars-ansi-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + - name: Prepare + run: | + dnf install -y python3.11 python3.11-pip python3.11-devel && \ + alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ + alternatives --set python3 /usr/bin/python3.11 && \ + pip3 install setuptools==77.0.3 && \ + pip3 install pyspark==3.5.5 cython && \ + pip3 install pandas==2.2.3 pyarrow==20.0.0 + - name: Prepare Spark Resources + run: | + rm -rf /opt/shims/spark41 + bash .github/workflows/util/install-resources.sh 4.1 + mv /opt/shims/spark41/spark_home/assembly/target/scala-2.12 /opt/shims/spark41/spark_home/assembly/target/scala-2.13 + - name: "Spark 4.1 spark-ut Tests (ANSI ON)" + run: | + set -o pipefail + cd $GITHUB_WORKSPACE/ + export SPARK_SCALA_VERSION=2.13 + yum install -y java-17-openjdk-devel + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE" + java -version + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ + -DwildcardSuites='org.apache.spark.' -Dtest=none -DfailIfNoTests=false \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ + 2>&1 | tee spark-ut-test-output.log + - name: "Parse test results" + if: always() + run: | + echo "=========================================" + echo " spark-ut (Spark 4.1, ANSI=$SPARK_ANSI_SQL_MODE)" + echo "=========================================" + echo "" + echo "--- Test Summary ---" + grep -E "Tests run:.*Failures:|BUILD " spark-ut-test-output.log || echo "(no summary found)" + echo "" + echo "--- Failed Tests ---" + grep -B1 "<<< FAIL!" spark-ut-test-output.log || echo "(no failures)" + echo "" + echo "--- Error Tests ---" + grep -B1 "<<< ERROR!" spark-ut-test-output.log || echo "(no errors)" + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: spark-test-spark-ut-ansi-report + path: | + **/surefire-reports/TEST-*.xml + spark-ut-test-output.log + - name: Upload log files + if: always() + uses: actions/upload-artifact@v4 + with: + name: spark-test-spark-ut-ansi-logs path: | **/target/*.log **/gluten-ut/**/hs_err_*.log --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
