This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit bc7f8e2a419a3b13e9b900d7c43a24534ea3306b Author: cxzl25 <[email protected]> AuthorDate: Fri Sep 11 13:43:14 2026 +0800 [spark] Convert EqualNullSafe(NaN) to isNaN in V2 filters (#9720) --- .../paimon/spark/SparkV2FilterConverter.scala | 2 ++ .../spark/sql/SparkV2FilterConverterTestBase.scala | 25 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala index 168c518cd3..41e5c47c4a 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala @@ -64,6 +64,8 @@ case class SparkV2FilterConverter(rowType: RowType) extends Logging { case BinaryPredicate(transform, literal) => if (literal == null) { builder.isNull(transform) + } else if (isNaN(literal)) { + builder.isNaN(transform) } else { PredicateBuilder.and(builder.isNotNull(transform), builder.equal(transform, literal)) } diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala index ea80fe476d..a865bce9ec 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala @@ -249,6 +249,31 @@ abstract class SparkV2FilterConverterTestBase extends PaimonSparkTestBase { sql("DROP TABLE IF EXISTS nan_tbl") } + test("V2Filter: EqualNullSafe NaN") { + sql("CREATE TABLE nan_tbl (id INT, f FLOAT, d DOUBLE) USING paimon") + sql("INSERT INTO nan_tbl VALUES (1, 1.0, 1.0)") + sql("INSERT INTO nan_tbl VALUES (2, CAST('NaN' AS FLOAT), CAST('NaN' AS DOUBLE))") + sql("INSERT INTO nan_tbl VALUES (3, 3.0, 3.0)") + + val nanRowType = loadTable("nan_tbl").rowType() + val nanBuilder = new PredicateBuilder(nanRowType) + val nanConverter = SparkV2FilterConverter(nanRowType) + + var filter = "f <=> CAST('NaN' AS FLOAT)" + var actual = nanConverter.convert(v2Filter(filter, "nan_tbl")).get + assert(actual.equals(nanBuilder.isNaN(1))) + checkAnswer(sql(s"SELECT id FROM nan_tbl WHERE $filter"), Seq(Row(2))) + assert(scanFilesCount(filter, "nan_tbl") >= 1) + + filter = "d <=> CAST('NaN' AS DOUBLE)" + actual = nanConverter.convert(v2Filter(filter, "nan_tbl")).get + assert(actual.equals(nanBuilder.isNaN(2))) + checkAnswer(sql(s"SELECT id FROM nan_tbl WHERE $filter"), Seq(Row(2))) + assert(scanFilesCount(filter, "nan_tbl") >= 1) + + sql("DROP TABLE IF EXISTS nan_tbl") + } + test("V2Filter: EqualNullSafe") { var filter = "int_col <=> 1" var actual = converter.convert(v2Filter(filter)).get
