This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new c99ce124b enable partial Spark SQL tests for iceberg compat (#1762)
c99ce124b is described below
commit c99ce124bedfaf7d0e7d14578d1cc90242be8a50
Author: Andy Grove <[email protected]>
AuthorDate: Wed May 21 10:31:51 2025 -0600
enable partial Spark SQL tests for iceberg compat (#1762)
---
.github/workflows/spark_sql_test.yml | 40 +++++++++++++++++++++++++++++++++++-
1 file changed, 39 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/spark_sql_test.yml
b/.github/workflows/spark_sql_test.yml
index f3a4c7576..9d9ac5f77 100644
--- a/.github/workflows/spark_sql_test.yml
+++ b/.github/workflows/spark_sql_test.yml
@@ -40,7 +40,7 @@ env:
RUST_VERSION: stable
jobs:
- spark-sql-catalyst:
+ spark-sql-native-comet:
strategy:
matrix:
os: [ubuntu-24.04]
@@ -79,3 +79,41 @@ jobs:
env:
LC_ALL: "C.UTF-8"
+ spark-sql-native-iceberg-compat:
+ strategy:
+ matrix:
+ os: [ubuntu-24.04]
+ java-version: [11]
+ spark-version: [{short: '3.5', full: '3.5.5'}]
+ module:
+ - {name: "catalyst", args1: "catalyst/test", args2: ""}
+# - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l
org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
+# - {name: "sql/core-2", args1: "", args2: "sql/testOnly * -- -n
org.apache.spark.tags.ExtendedSQLTest"}
+ - {name: "sql/core-3", args1: "", args2: "sql/testOnly * -- -n
org.apache.spark.tags.SlowSQLTest"}
+# - {name: "sql/hive-1", args1: "", args2: "hive/testOnly * -- -l
org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
+ - {name: "sql/hive-2", args1: "", args2: "hive/testOnly * -- -n
org.apache.spark.tags.ExtendedHiveTest"}
+ - {name: "sql/hive-3", args1: "", args2: "hive/testOnly * -- -n
org.apache.spark.tags.SlowHiveTest"}
+ fail-fast: false
+ name: spark-sql-iceberg-compat-${{ matrix.module.name }}/${{ matrix.os
}}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
+ runs-on: ${{ matrix.os }}
+ container:
+ image: amd64/rust
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup Rust & Java toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: ${{env.RUST_VERSION}}
+ jdk-version: ${{ matrix.java-version }}
+ - name: Setup Spark
+ uses: ./.github/actions/setup-spark-builder
+ with:
+ spark-version: ${{ matrix.spark-version.full }}
+ spark-short-version: ${{ matrix.spark-version.short }}
+ - name: Run Spark tests
+ run: |
+ cd apache-spark
+ rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet
cache requires cleanups
+ ENABLE_COMET=true ENABLE_COMET_SHUFFLE=true
COMET_SPARK_PARQUET_IMPL=native_iceberg_compat build/sbt ${{
matrix.module.args1 }} "${{ matrix.module.args2 }}"
+ env:
+ LC_ALL: "C.UTF-8"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
