This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 1e4e9f3a5a [VL] Split CI spark unit tests and TPC-DS random-kill job
across parallel runners (#12624)
1e4e9f3a5a is described below
commit 1e4e9f3a5a19c57d9f45f2eb588b7cd3f6afcfee
Author: Yuan <[email protected]>
AuthorDate: Mon Jul 27 09:23:46 2026 -0700
[VL] Split CI spark unit tests and TPC-DS random-kill job across parallel
runners (#12624)
* [VL] Split CI spark unit tests and TPC-DS random-kill job across parallel
runners
The Spark unit test jobs (spark-3.4 through spark-4.1) and the TPC-DS
random-kill job were running on a single runner each, taking up to ~2 hours
per job. After this the longest job is ~1 hour.
Changes:
- Split spark unit tests for
spark34/spark35/spark35-scala213/spark40/spark41
into 3 parallel runner groups using matrix strategy, partitioned by
package
prefix via -DwildcardSuites:
- Group 1: org.apache.spark.sql.streaming,
org.apache.spark.GlutenSortShuffleSuite,
org.apache.gluten
- Group 2: org.apache.spark.sql.execution,
org.apache.spark.sql.catalyst,
org.apache.spark.sql.errors, org.apache.spark.sql.extension
- Group 3: org.apache.spark.sql.GlutenSQL*,
org.apache.spark.sql.connector,
org.apache.spark.sql.sources, org.apache.spark.sql.hive,
org.apache.spark.sql.gluten, org.apache.spark.sql.shim
- Split the slow-test jobs (extended + hive) for each spark version into 2
parallel runners.
- Split tpc-test-ubuntu-randomkill into 3 parallel runners using the
built-in
--shard=N/M option in gluten-it, each runner handling ~33 of the 99 TPC-DS
queries.
Note: scalatest-maven-plugin uses -DwildcardSuites (not -Dtest) for suite
selection via package prefix.
Signed-off-by: Yuan <[email protected]>
* fix for enhanced branch
Signed-off-by: Yuan <[email protected]>
---------
Signed-off-by: Yuan <[email protected]>
---
.github/workflows/velox_backend_enhanced.yml | 27 ++-
.github/workflows/velox_backend_x86.yml | 256 +++++++++++++++------
.../spark/memory/GlobalOffHeapMemoryTarget.scala | 2 +-
3 files changed, 205 insertions(+), 80 deletions(-)
diff --git a/.github/workflows/velox_backend_enhanced.yml
b/.github/workflows/velox_backend_enhanced.yml
index a9e31d41f5..c51e9b777c 100644
--- a/.github/workflows/velox_backend_enhanced.yml
+++ b/.github/workflows/velox_backend_enhanced.yml
@@ -200,6 +200,10 @@ jobs:
spark-test-spark40:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ group: [1, 2, 3]
env:
SPARK_TESTING: true
container: apache/gluten:centos-8-jdk17
@@ -221,7 +225,7 @@ jobs:
run: |
rm -rf /opt/shims/spark40
bash .github/workflows/util/install-spark-resources.sh 4.0
- - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other
tests)
+ - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other
tests, group ${{ matrix.group }})
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.13
@@ -229,20 +233,31 @@ jobs:
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
- $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Piceberg -Pdelta \
- -Pspark-ut
-DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest
+
TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest"
+ if [ "${{ matrix.group }}" = "1" ]; then
+ $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Piceberg -Pdelta \
+ -Pspark-ut
-DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten"
+ elif [ "${{ matrix.group }}" = "2" ]; then
+ $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Piceberg -Pdelta \
+ -Pspark-ut
-DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension"
+ else
+ $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Piceberg -Pdelta \
+ -Pspark-ut
-DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim"
+ fi
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-report
+ name: spark-test-spark40-group${{ matrix.group }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-test-log
+ name: spark-test-spark40-group${{ matrix.group }}-test-log
path: |
**/target/*.log
**/gluten-ut/**/hs_err_*.log
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 9b13dbd069..d6fa89edf1 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -472,6 +472,7 @@ jobs:
fail-fast: false
matrix:
spark: [ "spark-3.3" ]
+ shard: [ 1, 2, 3 ]
runs-on: ubuntu-22.04
steps:
- name: Maximize build disk space
@@ -501,17 +502,17 @@ jobs:
echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV
- name: Build for Spark ${{ matrix.spark }}
run: |
- cd $GITHUB_WORKSPACE/
+ cd $GITHUB_WORKSPACE/
$MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox
-DskipTests
cd $GITHUB_WORKSPACE/tools/gluten-it
$GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }}
GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local
--benchmark-type=ds -s=30.0 --threads=12
- - name: TPC-DS SF30.0 Parquet local spark3.3 random kill tasks
+ - name: TPC-DS SF30.0 Parquet local spark3.3 random kill tasks (shard
${{ matrix.shard }}/3)
run: |
cd tools/gluten-it \
&& GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries \
--local --preset=velox --benchmark-type=ds --error-on-memleak
-s=30.0 --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1
\
- --data-gen=skip --random-kill-tasks --no-session-reuse
+ --data-gen=skip --random-kill-tasks --no-session-reuse
--shard=${{ matrix.shard }}/3
tpc-test-centos8-uniffle:
needs: build-native-lib-centos-7
@@ -737,6 +738,10 @@ jobs:
spark-test-spark34:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ group: [1, 2, 3]
container: apache/gluten:centos-8-jdk8
steps:
- uses: actions/checkout@v4
@@ -752,7 +757,7 @@ jobs:
pip3 install setuptools==77.0.3 && \
pip3 install pyspark==3.4.4 cython && \
pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Build and Run unit test for Spark 3.4.4 (other tests)
+ - name: Build and Run unit test for Spark 3.4.4 (other tests, group ${{
matrix.group }})
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
@@ -761,21 +766,31 @@ jobs:
export PATH=$JAVA_HOME/bin:$PATH
java -version
export SPARK_HOME=/opt/shims/spark34/spark_home/
- ls -l $SPARK_HOME
- $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg
-Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest
\
- -DargLine="-Dspark.test.home=$SPARK_HOME"
+
TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest"
+ if [ "${{ matrix.group }}" = "1" ]; then
+ $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox
-Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \
+ -DtagsToExclude="$TAGS_EXCLUDE"
-DargLine="-Dspark.test.home=$SPARK_HOME" \
+
-DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten"
+ elif [ "${{ matrix.group }}" = "2" ]; then
+ $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox
-Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \
+ -DtagsToExclude="$TAGS_EXCLUDE"
-DargLine="-Dspark.test.home=$SPARK_HOME" \
+
-DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension"
+ else
+ $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox
-Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \
+ -DtagsToExclude="$TAGS_EXCLUDE"
-DargLine="-Dspark.test.home=$SPARK_HOME" \
+
-DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim"
+ fi
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-report
+ name: spark-test-spark34-group${{ matrix.group }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-test-log
+ name: spark-test-spark34-group${{ matrix.group }}-test-log
path: |
**/target/*.log
**/gluten-ut/**/hs_err_*.log
@@ -784,12 +799,16 @@ jobs:
if: failure()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-golden-files
+ name: spark-test-spark34-group${{ matrix.group }}-golden-files
path: /tmp/tpch-approved-plan/**
spark-test-spark34-slow:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ suite: [extended, slow-hive]
env: #TODO remove after image update
LANG: C.UTF-8
LC_ALL: C.UTF-8
@@ -805,7 +824,7 @@ jobs:
run: |
rm -rf /opt/shims/spark34
bash .github/workflows/util/install-spark-resources.sh 3.4
- - name: Build and Run unit test for Spark 3.4.4 (slow tests)
+ - name: Build and Run unit test for Spark 3.4.4 (slow tests, ${{
matrix.suite }})
run: |
cd $GITHUB_WORKSPACE/
yum install -y java-17-openjdk-devel
@@ -813,24 +832,26 @@ jobs:
export PATH=$JAVA_HOME/bin:$PATH
java -version
export SPARK_HOME=/opt/shims/spark34/spark_home/
- ls -l $SPARK_HOME
- $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Phudi -Ppaimon -Pspark-ut \
- -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \
- -DargLine="-Dspark.test.home=$SPARK_HOME"
- $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Phudi -Ppaimon -Pspark-ut \
- -DtagsToInclude=org.apache.spark.tags.SlowHiveTest \
- -DargLine="-Dspark.test.home=$SPARK_HOME"
+ if [ "${{ matrix.suite }}" = "extended" ]; then
+ $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox
-Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
+ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \
+ -DargLine="-Dspark.test.home=$SPARK_HOME"
+ else
+ $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox
-Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
+ -DtagsToInclude=org.apache.spark.tags.SlowHiveTest \
+ -DargLine="-Dspark.test.home=$SPARK_HOME"
+ fi
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-report
+ name: spark-test-spark34-slow-${{ matrix.suite }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-test-log
+ name: spark-test-spark34-slow-${{ matrix.suite }}-test-log
path: |
**/target/*.log
**/gluten-ut/**/hs_err_*.log
@@ -839,6 +860,10 @@ jobs:
spark-test-spark35:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ group: [1, 2, 3]
env:
SPARK_TESTING: true
container: apache/gluten:centos-8-jdk8
@@ -856,7 +881,7 @@ jobs:
pip3 install setuptools==77.0.3 && \
pip3 install pyspark==3.5.5 cython && \
pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Build and Run unit test for Spark 3.5.5 (other tests)
+ - name: Build and Run unit test for Spark 3.5.5 (other tests, group ${{
matrix.group }})
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
@@ -864,20 +889,31 @@ jobs:
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
- $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Phudi -Ppaimon -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest
+
TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest"
+ if [ "${{ matrix.group }}" = "1" ]; then
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten"
+ elif [ "${{ matrix.group }}" = "2" ]; then
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension"
+ else
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim"
+ fi
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-report
+ name: spark-test-spark35-group${{ matrix.group }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-test-log
+ name: spark-test-spark35-group${{ matrix.group }}-test-log
path: |
**/target/*.log
**/gluten-ut/**/hs_err_*.log
@@ -886,12 +922,16 @@ jobs:
if: failure()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-golden-files
+ name: spark-test-spark35-group${{ matrix.group }}-golden-files
path: /tmp/tpch-approved-plan/**
spark-test-spark35-scala213:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ group: [1, 2, 3]
env:
SPARK_TESTING: true
container: apache/gluten:centos-8-jdk8
@@ -909,7 +949,7 @@ jobs:
pip3 install setuptools==77.0.3 && \
pip3 install pyspark==3.5.5 cython && \
pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Build and Run unit test for Spark 3.5.5 with scala-2.13 (other
tests)
+ - name: Build and Run unit test for Spark 3.5.5 with scala-2.13 (other
tests, group ${{ matrix.group }})
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.13
@@ -917,20 +957,31 @@ jobs:
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
- $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Piceberg \
- -Pdelta -Pspark-ut
-DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest
+
TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest"
+ if [ "${{ matrix.group }}" = "1" ]; then
+ $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Piceberg -Pdelta -Pspark-ut \
+
-DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten"
+ elif [ "${{ matrix.group }}" = "2" ]; then
+ $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Piceberg -Pdelta -Pspark-ut \
+
-DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension"
+ else
+ $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Piceberg -Pdelta -Pspark-ut \
+
-DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim"
+ fi
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-report
+ name: spark-test-spark35-scala213-group${{ matrix.group }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-test-log
+ name: spark-test-spark35-scala213-group${{ matrix.group }}-test-log
path: |
**/target/*.log
**/gluten-ut/**/hs_err_*.log
@@ -939,6 +990,10 @@ jobs:
spark-test-spark35-slow:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ suite: [extended, slow-hive]
env:
SPARK_TESTING: true
LANG: C.UTF-8 #TODO remove after image update
@@ -955,30 +1010,33 @@ jobs:
run: |
rm -rf /opt/shims/spark35
bash .github/workflows/util/install-spark-resources.sh 3.5
- - name: Build and Run unit test for Spark 3.5.5 (slow tests)
+ - name: Build and Run unit test for Spark 3.5.5 (slow tests, ${{
matrix.suite }})
run: |
cd $GITHUB_WORKSPACE/
yum install -y java-17-openjdk-devel
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
- $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Phudi -Ppaimon -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
- -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Phudi -Ppaimon -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
- -DtagsToInclude=org.apache.spark.tags.SlowHiveTest
+ if [ "${{ matrix.suite }}" = "extended" ]; then
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
+ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
+ else
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
+ -DtagsToInclude=org.apache.spark.tags.SlowHiveTest
+ fi
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-report
+ name: spark-test-spark35-slow-${{ matrix.suite }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-test-log
+ name: spark-test-spark35-slow-${{ matrix.suite }}-test-log
path: |
**/target/*.log
**/gluten-ut/**/hs_err_*.log
@@ -1130,6 +1188,10 @@ jobs:
spark-test-spark40:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ group: [1, 2, 3]
env:
SPARK_TESTING: true
container: apache/gluten:centos-8-jdk17
@@ -1151,7 +1213,7 @@ jobs:
run: |
rm -rf /opt/shims/spark40
bash .github/workflows/util/install-spark-resources.sh 4.0
- - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other
tests)
+ - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other
tests, group ${{ matrix.group }})
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.13
@@ -1159,20 +1221,31 @@ jobs:
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
- $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pdelta -Ppaimon \
- -Pspark-ut
-DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest
+
TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest"
+ if [ "${{ matrix.group }}" = "1" ]; then
+ $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten"
+ elif [ "${{ matrix.group }}" = "2" ]; then
+ $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension"
+ else
+ $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim"
+ fi
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-report
+ name: spark-test-spark40-group${{ matrix.group }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-test-log
+ name: spark-test-spark40-group${{ matrix.group }}-test-log
path: |
**/target/*.log
**/gluten-ut/**/hs_err_*.log
@@ -1181,6 +1254,10 @@ jobs:
spark-test-spark40-slow:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ suite: [extended, slow-hive]
env:
SPARK_TESTING: true
LANG: C.UTF-8 #TODO remove after image update
@@ -1197,30 +1274,33 @@ jobs:
run: |
rm -rf /opt/shims/spark40
bash .github/workflows/util/install-spark-resources.sh 4.0
- - name: Build and Run unit test for Spark 4.0 (slow tests)
+ - name: Build and Run unit test for Spark 4.0 (slow tests, ${{
matrix.suite }})
run: |
cd $GITHUB_WORKSPACE/
yum install -y java-17-openjdk-devel
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
- $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \
- -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pdelta -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \
- -DtagsToInclude=org.apache.spark.tags.SlowHiveTest
+ if [ "${{ matrix.suite }}" = "extended" ]; then
+ $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \
+ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
+ else
+ $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pdelta -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \
+ -DtagsToInclude=org.apache.spark.tags.SlowHiveTest
+ fi
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-report
+ name: spark-test-spark40-slow-${{ matrix.suite }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-test-log
+ name: spark-test-spark40-slow-${{ matrix.suite }}-test-log
path: |
**/target/*.log
**/gluten-ut/**/hs_err_*.log
@@ -1229,6 +1309,14 @@ jobs:
spark-test-spark41:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ # Split tests into 3 groups to run in parallel and cut the ~2h
wall-clock time.
+ # group1 – streaming tests
+ # group2 – execution / catalyst / errors / extension tests
+ # group3 – top-level sql, connector, sources, hive and remaining
tests
+ group: [1, 2, 3]
env:
SPARK_TESTING: true
container: apache/gluten:centos-9-jdk17
@@ -1252,7 +1340,7 @@ jobs:
run: |
rm -rf /opt/shims/spark41
bash .github/workflows/util/install-spark-resources.sh 4.1
- - name: Build and Run unit test for Spark 4.1.0 with scala-2.13 (other
tests)
+ - name: Build and Run unit test for Spark 4.1.0 with scala-2.13 (other
tests, group ${{ matrix.group }})
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.13
@@ -1260,20 +1348,34 @@ jobs:
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
- $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox \
- -Pspark-ut -Pdelta
-DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest
+
TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest"
+ if [ "${{ matrix.group }}" = "1" ]; then
+ # Group 1: streaming + gluten utils + top-level spark tests (~55
classes)
+ $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pspark-ut -Pdelta \
+ -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten"
+ elif [ "${{ matrix.group }}" = "2" ]; then
+ # Group 2: execution + catalyst + errors + extension tests (~140
classes)
+ $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pspark-ut -Pdelta \
+ -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension"
+ else
+ # Group 3: top-level sql, connector, sources, hive and remaining
tests (~205 classes)
+ $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pspark-ut -Pdelta \
+ -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/"
-DtagsToExclude="$TAGS_EXCLUDE" \
+
-DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim"
+ fi
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-report
+ name: spark-test-spark41-group${{ matrix.group }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-test-log
+ name: spark-test-spark41-group${{ matrix.group }}-test-log
path: |
**/target/*.log
**/gluten-ut/**/hs_err_*.log
@@ -1282,6 +1384,11 @@ jobs:
spark-test-spark41-slow:
needs: build-native-lib-centos-7
runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ # Run ExtendedSQLTest and SlowHiveTest in parallel instead of
sequentially.
+ suite: [extended, slow-hive]
env:
SPARK_TESTING: true
LANG: C.UTF-8 #TODO remove after image update
@@ -1298,30 +1405,33 @@ jobs:
run: |
rm -rf /opt/shims/spark41
bash .github/workflows/util/install-spark-resources.sh 4.1
- - name: Build and Run unit test for Spark 4.1 (slow tests)
+ - name: Build and Run unit test for Spark 4.1 (slow tests, ${{
matrix.suite }})
run: |
cd $GITHUB_WORKSPACE/
yum install -y java-17-openjdk-devel
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
- $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \
- -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \
- -DtagsToInclude=org.apache.spark.tags.SlowHiveTest
+ if [ "${{ matrix.suite }}" = "extended" ]; then
+ $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \
+ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
+ else
+ $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \
+ -DtagsToInclude=org.apache.spark.tags.SlowHiveTest
+ fi
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-report
+ name: spark-test-spark41-slow-${{ matrix.suite }}-report
path: '**/surefire-reports/TEST-*.xml'
- name: Upload unit tests log files
if: ${{ !success() }}
uses: actions/upload-artifact@v4
with:
- name: ${{ github.job }}-test-log
+ name: spark-test-spark41-slow-${{ matrix.suite }}-test-log
path: |
**/target/*.log
**/gluten-ut/**/hs_err_*.log
diff --git
a/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala
b/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala
index ab73aa40fa..96749057fe 100644
---
a/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala
+++
b/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala
@@ -35,7 +35,7 @@ class GlobalOffHeapMemoryTarget private[memory]
with Logging {
private val targetName = MemoryTargetUtil.toUniqueName("GlobalOffHeap")
private val recorder: MemoryUsageRecorder = new SimpleMemoryUsageRecorder()
- private val mode: MemoryMode = {
+ private def mode: MemoryMode = {
val enabled = Option(SparkEnv.get)
.map(_.conf.getBoolean(GlutenCoreConfig.DYNAMIC_OFFHEAP_SIZING_ENABLED.key,
false))
.getOrElse(false)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
