This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch feature/ansi-ci-baseline in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 82ac3a3aa9c4da51f247e63592880e2bfe632822 Author: baibaichen <[email protected]> AuthorDate: Fri Mar 27 00:34:44 2026 +0800 ci: add ANSI mode baseline workflow for Spark 4.1 Add velox_backend_ansi.yml with SPARK_ANSI_SQL_MODE=true to collect ANSI test baseline for Spark 4.1. Supports workflow_dispatch toggle to switch between ANSI ON (baseline) and ANSI OFF (regression check). This workflow runs on personal fork only, not merged to upstream. Issue: https://github.com/apache/gluten/issues/10134 --- .github/workflows/velox_backend_ansi.yml | 175 +++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/.github/workflows/velox_backend_ansi.yml b/.github/workflows/velox_backend_ansi.yml new file mode 100644 index 0000000000..97c21ac3a9 --- /dev/null +++ b/.github/workflows/velox_backend_ansi.yml @@ -0,0 +1,175 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Velox Backend ANSI Mode + +on: + workflow_dispatch: + inputs: + ansi_mode: + description: 'ANSI mode: true=ANSI baseline, false=regression check' + required: false + default: 'true' + schedule: + - cron: '0 8 * * *' + pull_request: + paths: + - '.github/workflows/velox_backend_ansi.yml' + - '.github/workflows/util/install-resources.sh' + - 'pom.xml' + - 'backends-velox/**' + - 'gluten-core/**' + - 'gluten-substrait/**' + - 'gluten-arrow/**' + - 'gluten-ut/**' + - 'shims/**' + - 'ep/build-velox/**' + - 'cpp/**' + - 'dev/**' + - 'build/mvn' + +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + MVN_CMD: 'build/mvn -ntp' + WGET_CMD: 'wget -nv' + CCACHE_DIR: "${{ github.workspace }}/.ccache" + # workflow_dispatch 手动输入时可切换，schedule/pull_request 默认 true + SPARK_ANSI_SQL_MODE: ${{ github.event.inputs.ansi_mode || 'true' }} + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build-native-lib: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Get Ccache + uses: actions/cache/restore@v4 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos7-release-default-${{github.sha}} + restore-keys: | + ccache-centos7-release-default + - name: Build Gluten native libraries + run: | + docker pull apache/gluten:vcpkg-centos-7 + docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c " + set -e + yum install tzdata -y + cd /work + export CCACHE_DIR=/work/.ccache + export CCACHE_MAXSIZE=1G + mkdir -p /work/.ccache + ccache -sz + bash dev/ci-velox-buildstatic-centos-7.sh + ccache -s + mkdir -p /work/.m2/repository/org/apache/arrow/ + cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/ + " + - name: Save ccache + if: always() + uses: actions/cache/save@v4 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos7-release-default-${{github.sha}} + - uses: actions/upload-artifact@v4 + with: + name: velox-native-lib-ansi-${{github.sha}} + path: ./cpp/build/ + if-no-files-found: error + - uses: actions/upload-artifact@v4 + with: + name: arrow-jars-ansi-${{github.sha}} + path: .m2/repository/org/apache/arrow/ + if-no-files-found: error + + spark-test-spark41-ansi: + needs: build-native-lib + runs-on: ubuntu-22.04 + env: + SPARK_TESTING: true + container: apache/gluten:centos-9-jdk17 + steps: + - uses: actions/checkout@v4 + - name: Download Native Lib + uses: actions/download-artifact@v4 + with: + name: velox-native-lib-ansi-${{github.sha}} + path: ./cpp/build/ + - name: Download Arrow Jars + uses: actions/download-artifact@v4 + with: + name: arrow-jars-ansi-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + - name: Prepare + run: | + dnf install -y python3.11 python3.11-pip python3.11-devel && \ + alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ + alternatives --set python3 /usr/bin/python3.11 && \ + pip3 install setuptools==77.0.3 && \ + pip3 install pyspark==3.5.5 cython && \ + pip3 install pandas==2.2.3 pyarrow==20.0.0 + - name: Prepare Spark Resources + run: | + rm -rf /opt/shims/spark41 + bash .github/workflows/util/install-resources.sh 4.1 + mv /opt/shims/spark41/spark_home/assembly/target/scala-2.12 /opt/shims/spark41/spark_home/assembly/target/scala-2.13 + - name: "Spark 4.1 Unit Tests (ANSI=${{ env.SPARK_ANSI_SQL_MODE }})" + run: | + cd $GITHUB_WORKSPACE/ + export SPARK_SCALA_VERSION=2.13 + yum install -y java-17-openjdk-devel + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE" + java -version + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ + -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ + 2>&1 | tee ansi-test-output.log || true + - name: "Parse test results" + if: always() + run: | + echo "=========================================" + echo " Test Results (Spark 4.1, ANSI=$SPARK_ANSI_SQL_MODE)" + echo "=========================================" + echo "" + echo "--- Test Summary ---" + grep -E "Tests run:.*Failures:|BUILD " ansi-test-output.log || echo "(no summary found)" + echo "" + echo "--- Failed Tests ---" + grep -B1 "<<< FAIL!" ansi-test-output.log || echo "(no failures)" + echo "" + echo "--- Error Tests ---" + grep -B1 "<<< ERROR!" ansi-test-output.log || echo "(no errors)" + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: spark-test-spark41-ansi-report + path: | + **/surefire-reports/TEST-*.xml + ansi-test-output.log + - name: Upload log files + if: always() + uses: actions/upload-artifact@v4 + with: + name: spark-test-spark41-ansi-logs + path: | + **/target/*.log + **/gluten-ut/**/hs_err_*.log + **/gluten-ut/**/core.* --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
