This is an automated email from the ASF dual-hosted git repository.
kazuyukitanimura pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 2c1a6b9c fix: Spark 4.0-preview1 SPARK-47120 (#1156)
2c1a6b9c is described below
commit 2c1a6b9c576042f2e0a73cf76a3ad93db46b6a0b
Author: KAZUYUKI TANIMURA <[email protected]>
AuthorDate: Wed Dec 11 09:44:12 2024 -0800
fix: Spark 4.0-preview1 SPARK-47120 (#1156)
## Which issue does this PR close?
Part of https://github.com/apache/datafusion-comet/issues/372 and
https://github.com/apache/datafusion-comet/issues/551
## Rationale for this change
To be ready for Spark 4.0
## What changes are included in this PR?
This PR fixes the new test SPARK-47120 added in Spark 4.0
## How are these changes tested?
tests enabled
---
dev/diffs/4.0.0-preview1.diff | 16 +++-------------
docs/source/user-guide/source.md | 2 +-
.../scala/org/apache/comet/parquet/ParquetFilters.scala | 10 ++++++----
3 files changed, 10 insertions(+), 18 deletions(-)
diff --git a/dev/diffs/4.0.0-preview1.diff b/dev/diffs/4.0.0-preview1.diff
index 1a6810a7..ba68d2a7 100644
--- a/dev/diffs/4.0.0-preview1.diff
+++ b/dev/diffs/4.0.0-preview1.diff
@@ -2088,7 +2088,7 @@ index cd6f41b4ef4..4b6a17344bc 100644
ParquetOutputFormat.WRITER_VERSION ->
ParquetProperties.WriterVersion.PARQUET_2_0.toString
)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 795e9f46a8d..6285a1e388b 100644
+index 795e9f46a8d..5306c94a686 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -1100,7 +1100,11 @@ abstract class ParquetFilterSuite extends QueryTest
with ParquetTest with Shared
@@ -2150,17 +2150,7 @@ index 795e9f46a8d..6285a1e388b 100644
// block 1:
// null count min
max
// page-0 0 0
99
-@@ -2211,7 +2225,8 @@ abstract class ParquetFilterSuite extends QueryTest with
ParquetTest with Shared
- }
- }
-
-- test("SPARK-47120: subquery literal filter pushdown") {
-+ test("SPARK-47120: subquery literal filter pushdown",
-+ IgnoreComet("TODO:
https://github.com/apache/datafusion-comet/issues/551";)) {
- withTable("t1", "t2") {
- sql("create table t1(d date) using parquet")
- sql("create table t2(d date) using parquet")
-@@ -2301,7 +2316,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
+@@ -2301,7 +2315,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
assert(pushedParquetFilters.exists(_.getClass === filterClass),
s"${pushedParquetFilters.map(_.getClass).toList} did not contain
${filterClass}.")
@@ -2173,7 +2163,7 @@ index 795e9f46a8d..6285a1e388b 100644
} else {
assert(selectedFilters.isEmpty, "There is filter pushed down")
}
-@@ -2362,7 +2381,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
+@@ -2362,7 +2380,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
assert(pushedParquetFilters.exists(_.getClass === filterClass),
s"${pushedParquetFilters.map(_.getClass).toList} did not contain
${filterClass}.")
diff --git a/docs/source/user-guide/source.md b/docs/source/user-guide/source.md
index 71c9060c..5f5d03d9 100644
--- a/docs/source/user-guide/source.md
+++ b/docs/source/user-guide/source.md
@@ -27,7 +27,7 @@ Official source releases can be downloaded from
https://dist.apache.org/repos/di
```console
# Pick the latest version
-export COMET_VERSION=0.3.0
+export COMET_VERSION=0.4.0
# Download the tarball
curl -O
"https://dist.apache.org/repos/dist/release/datafusion/datafusion-comet-$COMET_VERSION/apache-datafusion-comet-$COMET_VERSION.tar.gz";
# Unpack
diff --git a/spark/src/main/scala/org/apache/comet/parquet/ParquetFilters.scala
b/spark/src/main/scala/org/apache/comet/parquet/ParquetFilters.scala
index 17844aba..bcb23986 100644
--- a/spark/src/main/scala/org/apache/comet/parquet/ParquetFilters.scala
+++ b/spark/src/main/scala/org/apache/comet/parquet/ParquetFilters.scala
@@ -723,20 +723,22 @@ class ParquetFilters(
.lift(nameToParquetField(name).fieldType)
.map(_(nameToParquetField(name).fieldNames, value))
- case sources.LessThan(name, value) if canMakeFilterOn(name, value) =>
+ case sources.LessThan(name, value) if (value != null) &&
canMakeFilterOn(name, value) =>
makeLt
.lift(nameToParquetField(name).fieldType)
.map(_(nameToParquetField(name).fieldNames, value))
- case sources.LessThanOrEqual(name, value) if canMakeFilterOn(name,
value) =>
+ case sources.LessThanOrEqual(name, value)
+ if (value != null) && canMakeFilterOn(name, value) =>
makeLtEq
.lift(nameToParquetField(name).fieldType)
.map(_(nameToParquetField(name).fieldNames, value))
- case sources.GreaterThan(name, value) if canMakeFilterOn(name, value) =>
+ case sources.GreaterThan(name, value) if (value != null) &&
canMakeFilterOn(name, value) =>
makeGt
.lift(nameToParquetField(name).fieldType)
.map(_(nameToParquetField(name).fieldNames, value))
- case sources.GreaterThanOrEqual(name, value) if canMakeFilterOn(name,
value) =>
+ case sources.GreaterThanOrEqual(name, value)
+ if (value != null) && canMakeFilterOn(name, value) =>
makeGtEq
.lift(nameToParquetField(name).fieldType)
.map(_(nameToParquetField(name).fieldNames, value))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
