This is an automated email from the ASF dual-hosted git repository.
mbutrovich pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new b8c5275f5 build: reduce overhead of fuzz testing (#3257)
b8c5275f5 is described below
commit b8c5275f5e7a9682f6e9efc818b55c91db510add
Author: Andy Grove <[email protected]>
AuthorDate: Mon Jan 26 09:55:29 2026 -0700
build: reduce overhead of fuzz testing (#3257)
* build: reduce overhead of fuzz testing
* make workflow names consistent
* fix
---
.github/workflows/pr_build_linux.yml | 10 +++++-----
.../test/scala/org/apache/comet/CometFuzzTestBase.scala | 17 +++++------------
2 files changed, 10 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/pr_build_linux.yml
b/.github/workflows/pr_build_linux.yml
index ac100944a..7df0aa069 100644
--- a/.github/workflows/pr_build_linux.yml
+++ b/.github/workflows/pr_build_linux.yml
@@ -169,19 +169,19 @@ jobs:
- name: "Spark 3.5.5, JDK 17, Scala 2.13"
java_version: "17"
maven_opts: "-Pspark-3.5 -Dspark.version=3.5.5 -Pscala-2.13"
- scan_impl: "native_comet"
+ scan_impl: "auto"
- name: "Spark 3.5.6, JDK 17, Scala 2.13"
java_version: "17"
maven_opts: "-Pspark-3.5 -Dspark.version=3.5.6 -Pscala-2.13"
scan_impl: "native_comet"
- - name: "Spark 3.5, JDK 17, Scala 2.12 native_datafusion"
+ - name: "Spark 3.5, JDK 17, Scala 2.12"
java_version: "17"
maven_opts: "-Pspark-3.5 -Pscala-2.12"
scan_impl: "native_datafusion"
- - name: "Spark 3.5, JDK 17, Scala 2.12 native_iceberg_compat"
+ - name: "Spark 3.5, JDK 17, Scala 2.12"
java_version: "17"
maven_opts: "-Pspark-3.5 -Pscala-2.12"
scan_impl: "native_iceberg_compat"
@@ -189,7 +189,7 @@ jobs:
- name: "Spark 4.0, JDK 17"
java_version: "17"
maven_opts: "-Pspark-4.0"
- scan_impl: "native_comet"
+ scan_impl: "auto"
suite:
- name: "fuzz"
value: |
@@ -267,7 +267,7 @@ jobs:
value: |
org.apache.spark.sql.CometToPrettyStringSuite
fail-fast: false
- name: ${{ matrix.os }}/${{ matrix.profile.name }} [${{ matrix.suite.name
}}]
+ name: ${{ matrix.os }}/${{ matrix.profile.name }}/${{
matrix.profile.scan_impl }} [${{ matrix.suite.name }}]
runs-on: ${{ matrix.os }}
container:
image: amd64/rust
diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala
b/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala
index 5c5251b5e..0e179b40f 100644
--- a/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala
+++ b/spark/src/test/scala/org/apache/comet/CometFuzzTestBase.scala
@@ -112,18 +112,11 @@ class CometFuzzTestBase extends CometTestBase with
AdaptiveSparkPlanHelper {
override protected def test(testName: String, testTags: Tag*)(testFun: =>
Any)(implicit
pos: Position): Unit = {
Seq("native", "jvm").foreach { shuffleMode =>
- Seq(
- CometConf.SCAN_AUTO,
- CometConf.SCAN_NATIVE_COMET,
- CometConf.SCAN_NATIVE_DATAFUSION,
- CometConf.SCAN_NATIVE_ICEBERG_COMPAT).foreach { scanImpl =>
- super.test(testName + s" ($scanImpl, $shuffleMode shuffle)", testTags:
_*) {
- withSQLConf(
- CometConf.COMET_NATIVE_SCAN_IMPL.key -> scanImpl,
- CometConf.COMET_PARQUET_UNSIGNED_SMALL_INT_CHECK.key -> "false",
- CometConf.COMET_SHUFFLE_MODE.key -> shuffleMode) {
- testFun
- }
+ super.test(testName + s" ($shuffleMode shuffle)", testTags: _*) {
+ withSQLConf(
+ CometConf.COMET_PARQUET_UNSIGNED_SMALL_INT_CHECK.key -> "false",
+ CometConf.COMET_SHUFFLE_MODE.key -> shuffleMode) {
+ testFun
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
