This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 4100c25b34 [VL] fix nightly build; adding spark-4.2 to nightly build
(#13327)
4100c25b34 is described below
commit 4100c25b34e53d966cb5fae7f2944c3d1e5a1388
Author: Yuan <[email protected]>
AuthorDate: Fri Oct 23 09:17:26 2026 +0100
[VL] fix nightly build; adding spark-4.2 to nightly build (#13327)
---
.github/workflows/velox_nightly.yml | 199 +++++++++++----------
.../gluten/execution/VeloxIcebergSuite.scala | 12 +-
2 files changed, 111 insertions(+), 100 deletions(-)
diff --git a/.github/workflows/velox_nightly.yml
b/.github/workflows/velox_nightly.yml
index 82f68d4858..3fc4917462 100644
--- a/.github/workflows/velox_nightly.yml
+++ b/.github/workflows/velox_nightly.yml
@@ -135,6 +135,7 @@ jobs:
./build/mvn clean install -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ ./build/mvn clean install -Pspark-4.2 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v7
with:
@@ -170,6 +171,7 @@ jobs:
cd $GITHUB_WORKSPACE/ && \
./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -DskipTests
-Dmaven.source.skip
+ ./build/mvn clean install -Pspark-4.2 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v7
with:
@@ -177,103 +179,6 @@ jobs:
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7
- spark-test-spark35-smj:
- if: ${{ startsWith(github.repository, 'apache/') }}
- needs: build-native-lib-x86
- runs-on: ubuntu-22.04
- env:
- SPARK_TESTING: true
- container: apache/gluten:centos-8-jdk8
- steps:
- - uses: actions/checkout@v4
- - name: Download All Artifacts
- uses: actions/download-artifact@v4
- with:
- name: velox-native-lib-${{github.sha}}
- path: ./cpp/build/releases
- - name: Prepare
- run: |
- dnf module -y install python39 && \
- alternatives --set python3 /usr/bin/python3.9 && \
- pip3 install setuptools==77.0.3 && \
- pip3 install pyspark==3.5.5 cython && \
- pip3 install pandas==2.2.3 pyarrow==20.0.0
- - name: Build and Run unit test for Spark 3.5.5 (other tests)
- run: |
- cd $GITHUB_WORKSPACE/
- export SPARK_SCALA_VERSION=2.12
- yum install -y java-17-openjdk-devel
- export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
- export PATH=$JAVA_HOME/bin:$PATH
- java -version
- $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Ppaimon -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
-
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest
- - name: Upload test report
- if: always()
- uses: actions/upload-artifact@v4
- with:
- name: ${{ github.job }}-report
- path: '**/surefire-reports/TEST-*.xml'
- - name: Upload unit tests log files
- if: ${{ !success() }}
- uses: actions/upload-artifact@v4
- with:
- name: ${{ github.job }}-test-log
- path: |
- **/target/*.log
- **/gluten-ut/**/hs_err_*.log
- **/gluten-ut/**/core.*
-
- spark-test-spark35-slow-smj:
- if: ${{ startsWith(github.repository, 'apache/') }}
- needs: build-native-lib-x86
- runs-on: ubuntu-22.04
- env:
- SPARK_TESTING: true
- LANG: C.UTF-8 #TODO remove after image update
- LC_ALL: C.UTF-8
- container: apache/gluten:centos-8-jdk8
- steps:
- - uses: actions/checkout@v4
- - name: Download All Artifacts
- uses: actions/download-artifact@v4
- with:
- name: velox-native-lib-${{github.sha}}
- path: ./cpp/build/releases
- - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image
update
- run: |
- rm -rf /opt/shims/spark35
- bash .github/workflows/util/install-spark-resources.sh 3.5
- - name: Build and Run unit test for Spark 3.5.5 (slow tests)
- run: |
- cd $GITHUB_WORKSPACE/
- yum install -y java-17-openjdk-devel
- export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
- export PATH=$JAVA_HOME/bin:$PATH
- java -version
- $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Ppaimon -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
- -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Ppaimon -Pspark-ut \
- -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
- -DtagsToInclude=org.apache.spark.tags.SlowHiveTest
- - name: Upload test report
- if: always()
- uses: actions/upload-artifact@v4
- with:
- name: ${{ github.job }}-report
- path: '**/surefire-reports/TEST-*.xml'
- - name: Upload unit tests log files
- if: ${{ !success() }}
- uses: actions/upload-artifact@v4
- with:
- name: ${{ github.job }}-test-log
- path: |
- **/target/*.log
- **/gluten-ut/**/hs_err_*.log
- **/gluten-ut/**/core.*
-
#build and package for arm64
build-native-lib-centos-8-arm64:
if: ${{ startsWith(github.repository, 'apache/') }}
@@ -376,6 +281,7 @@ jobs:
./build/mvn clean install -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ ./build/mvn clean install -Pspark-4.2 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v7
with:
@@ -411,6 +317,7 @@ jobs:
cd $GITHUB_WORKSPACE/ && \
./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ ./build/mvn clean install -Pspark-4.2 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v7
with:
@@ -576,6 +483,7 @@ jobs:
./build/mvn clean install -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
+ ./build/mvn clean install -Pspark-4.2 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v7
with:
@@ -592,3 +500,100 @@ jobs:
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
+
+ spark-test-spark35-smj:
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ needs: build-native-lib-x86
+ runs-on: ubuntu-22.04
+ env:
+ SPARK_TESTING: true
+ container: apache/gluten:centos-8-jdk8
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Prepare
+ run: |
+ dnf module -y install python39 && \
+ alternatives --set python3 /usr/bin/python3.9 && \
+ pip3 install setuptools==77.0.3 && \
+ pip3 install pyspark==3.5.5 cython && \
+ pip3 install pandas==2.2.3 pyarrow==20.0.0
+ - name: Build and Run unit test for Spark 3.5.5 (other tests)
+ run: |
+ cd $GITHUB_WORKSPACE/
+ export SPARK_SCALA_VERSION=2.12
+ yum install -y java-17-openjdk-devel
+ export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ export PATH=$JAVA_HOME/bin:$PATH
+ java -version
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
+
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.CudfTest,org.apache.gluten.tags.SkipTest
+ - name: Upload test report
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ github.job }}-report
+ path: '**/surefire-reports/TEST-*.xml'
+ - name: Upload unit tests log files
+ if: ${{ !success() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ github.job }}-test-log
+ path: |
+ **/target/*.log
+ **/gluten-ut/**/hs_err_*.log
+ **/gluten-ut/**/core.*
+
+ spark-test-spark35-slow-smj:
+ if: ${{ startsWith(github.repository, 'apache/') }}
+ needs: build-native-lib-x86
+ runs-on: ubuntu-22.04
+ env:
+ SPARK_TESTING: true
+ LANG: C.UTF-8 #TODO remove after image update
+ LC_ALL: C.UTF-8
+ container: apache/gluten:centos-8-jdk8
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download All Artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: velox-native-lib-${{github.sha}}
+ path: ./cpp/build/releases
+ - name: Prepare Spark Resources for Spark 3.5.5 #TODO remove after image
update
+ run: |
+ rm -rf /opt/shims/spark35
+ bash .github/workflows/util/install-spark-resources.sh 3.5
+ - name: Build and Run unit test for Spark 3.5.5 (slow tests)
+ run: |
+ cd $GITHUB_WORKSPACE/
+ yum install -y java-17-openjdk-devel
+ export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ export PATH=$JAVA_HOME/bin:$PATH
+ java -version
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
+ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Ppaimon -Pspark-ut \
+ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
+ -DtagsToInclude=org.apache.spark.tags.SlowHiveTest
+ - name: Upload test report
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ github.job }}-report
+ path: '**/surefire-reports/TEST-*.xml'
+ - name: Upload unit tests log files
+ if: ${{ !success() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ github.job }}-test-log
+ path: |
+ **/target/*.log
+ **/gluten-ut/**/hs_err_*.log
+ **/gluten-ut/**/core.*
diff --git
a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
index 4b71e6d4b6..6ff43d256c 100644
---
a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
+++
b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
@@ -29,7 +29,7 @@ import org.apache.iceberg.{FileScanTask, UpdateSchema}
import org.apache.iceberg.expressions.Literal
import org.apache.iceberg.hadoop.HadoopCatalog
import org.apache.iceberg.io.{ResolvingFileIO, StorageCredential}
-import org.apache.iceberg.spark.{ScanTaskSetManager, Spark3Util, SparkCatalog,
SparkReadOptions}
+import org.apache.iceberg.spark.{ScanTaskSetManager, Spark3Util, SparkCatalog,
SparkTableCache}
import org.apache.iceberg.spark.source.{GlutenIcebergSourceUtil, SparkTable}
import org.apache.iceberg.types.{Type, Types}
@@ -120,15 +120,20 @@ class VeloxIcebergSuite extends IcebergSuite {
val table = Spark3Util.loadIcebergTable(spark, tableName)
val taskSetId = "vended-credentials"
val taskSetManager = ScanTaskSetManager.get()
+ val tableCache = SparkTableCache.get()
val tasks = table.newScan().planFiles()
try {
+ // Register the table under the task set ID and load by that ID,
which resolves to a
+ // staged scan on both Iceberg 1.10 (needs the "scan-task-set-id"
option) and Iceberg
+ // 1.11+ (removed SparkReadOptions.SCAN_TASK_SET_ID, resolves cached
tables directly).
+ tableCache.add(taskSetId, table)
taskSetManager.stageTasks(table, taskSetId,
tasks.asScala.toList.asJava)
compareDfResultsAgainstVanillaSpark(
() =>
spark.read
.format("iceberg")
- .option(SparkReadOptions.SCAN_TASK_SET_ID, taskSetId)
- .load(tableName),
+ .option("scan-task-set-id", taskSetId)
+ .load(taskSetId),
noFallBack = false,
customCheck = df => {
checkAnswer(df, Seq(Row(1), Row(2)))
@@ -142,6 +147,7 @@ class VeloxIcebergSuite extends IcebergSuite {
}
)
} finally {
+ tableCache.remove(taskSetId)
taskSetManager.removeTasks[FileScanTask](table, taskSetId)
tasks.close()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
