This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1009904c33 [codegen] Fix equaliser treating null array elements and
map values as unequal (#9860)
1009904c33 is described below
commit 1009904c334a39f1cf3540b1d1122464f9c4f08c
Author: Eunbin Son <[email protected]>
AuthorDate: Thu Sep 17 15:45:27 2026 +0900
[codegen] Fix equaliser treating null array elements and map values as
unequal (#9860)
---
.../apache/paimon/codegen/ScalarOperatorGens.scala | 17 ++++-
.../paimon/codegen/EqualiserCodeGeneratorTest.java | 78 ++++++++++++++++++++++
2 files changed, 92 insertions(+), 3 deletions(-)
diff --git
a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala
b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala
index fa46e3d02a..2397b0e060 100644
---
a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala
+++
b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala
@@ -272,7 +272,7 @@ object ScalarOperatorGens {
| }
|
| ${elementEqualsExpr.code}
- | if (!${elementEqualsExpr.resultTerm}) {
+ | if (!${nullSafeEquals(leftElementNullTerm,
rightElementNullTerm, elementEqualsExpr.resultTerm)}) {
| $resultTerm = false;
| break;
| }
@@ -396,7 +396,7 @@ object ScalarOperatorGens {
| }
|
| ${valueEqualsExpr.code}
- | if (${valueEqualsExpr.resultTerm}) {
+ | if (${nullSafeEquals(leftValueNullTerm,
rightValueNullTerm, valueEqualsExpr.resultTerm)}) {
| $matchedTerm[$rightIndexTerm] = true;
| $foundTerm = true;
| }
@@ -432,7 +432,7 @@ object ScalarOperatorGens {
| boolean $rightValueNullTerm = ($rightValueTerm == null);
|
| ${valueEqualsExpr.code}
- | if (!${valueEqualsExpr.resultTerm}) {
+ | if (!${nullSafeEquals(leftValueNullTerm,
rightValueNullTerm, valueEqualsExpr.resultTerm)}) {
| $resultTerm = false;
| break;
| }
@@ -464,6 +464,17 @@ object ScalarOperatorGens {
}
}
+ /**
+ * Null-safe equality on a nested element or value: two nulls are equal, a
single null is not,
+ * otherwise the generated equality decides. `generateEquals` leaves its
result flag `false` when
+ * either operand is null, so reading that flag alone treats two null
elements as unequal.
+ */
+ private def nullSafeEquals(
+ leftNullTerm: String,
+ rightNullTerm: String,
+ equalsTerm: String): String =
+ s"(($leftNullTerm && $rightNullTerm) || (!$leftNullTerm && !$rightNullTerm
&& $equalsTerm))"
+
private def containsFloatingPoint(t: DataType): Boolean = t.getTypeRoot
match {
case FLOAT | DOUBLE => true
case DataTypeRoot.ARRAY | DataTypeRoot.MAP | DataTypeRoot.MULTISET |
DataTypeRoot.ROW |
diff --git
a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java
b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java
index ab4ad74af8..fc0716903c 100644
---
a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java
+++
b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java
@@ -571,6 +571,84 @@ public class EqualiserCodeGeneratorTest {
false);
}
+ @Test
+ public void testArrayEqualiserTreatsNullElementsAsEqual() {
+ DataType intArrayType = DataTypes.ARRAY(DataTypes.INT());
+ RecordEqualiser intArrayEqualiser =
+ new EqualiserCodeGenerator(new DataType[] {intArrayType})
+
.generateRecordEqualiser("nullElementIntArrayFieldEquals")
+
.newInstance(Thread.currentThread().getContextClassLoader());
+ assertFloatingPointEqualiser(
+ intArrayEqualiser,
+ intArrayType,
+ new GenericArray(new Object[] {1, null}),
+ new GenericArray(new Object[] {1, null}),
+ true);
+ assertFloatingPointEqualiser(
+ intArrayEqualiser,
+ intArrayType,
+ new GenericArray(new Object[] {1, null}),
+ new GenericArray(new Object[] {1, 2}),
+ false);
+
+ DataType doubleArrayType = DataTypes.ARRAY(DataTypes.DOUBLE());
+ RecordEqualiser doubleArrayEqualiser =
+ new EqualiserCodeGenerator(new DataType[] {doubleArrayType})
+
.generateRecordEqualiser("nullElementDoubleArrayFieldEquals")
+
.newInstance(Thread.currentThread().getContextClassLoader());
+ assertFloatingPointEqualiser(
+ doubleArrayEqualiser,
+ doubleArrayType,
+ new GenericArray(new Object[] {1.0d, null}),
+ new GenericArray(new Object[] {1.0d, null}),
+ true);
+
+ DataType rowArrayType =
DataTypes.ARRAY(DataTypes.ROW(DataTypes.INT()));
+ RecordEqualiser rowArrayEqualiser =
+ new EqualiserCodeGenerator(new DataType[] {rowArrayType})
+
.generateRecordEqualiser("nullElementRowArrayFieldEquals")
+
.newInstance(Thread.currentThread().getContextClassLoader());
+ assertFloatingPointEqualiser(
+ rowArrayEqualiser,
+ rowArrayType,
+ new GenericArray(new Object[] {null}),
+ new GenericArray(new Object[] {null}),
+ true);
+ }
+
+ @Test
+ public void testMapEqualiserTreatsNullValuesAsEqual() {
+ DataType intKeyMapType = DataTypes.MAP(DataTypes.INT(),
DataTypes.INT());
+ RecordEqualiser intKeyMapEqualiser =
+ new EqualiserCodeGenerator(new DataType[] {intKeyMapType})
+
.generateRecordEqualiser("nullValueIntKeyMapFieldEquals")
+
.newInstance(Thread.currentThread().getContextClassLoader());
+ assertMapEqualiser(
+ intKeyMapEqualiser,
+ intKeyMapType,
+ new GenericMap(singletonMap(1, null)),
+ new GenericMap(singletonMap(1, null)),
+ true);
+ assertMapEqualiser(
+ intKeyMapEqualiser,
+ intKeyMapType,
+ new GenericMap(singletonMap(1, null)),
+ new GenericMap(singletonMap(1, 2)),
+ false);
+
+ DataType binaryKeyMapType = DataTypes.MAP(DataTypes.BYTES(),
DataTypes.INT());
+ RecordEqualiser binaryKeyMapEqualiser =
+ new EqualiserCodeGenerator(new DataType[] {binaryKeyMapType})
+
.generateRecordEqualiser("nullValueBinaryKeyMapFieldEquals")
+
.newInstance(Thread.currentThread().getContextClassLoader());
+ assertMapEqualiser(
+ binaryKeyMapEqualiser,
+ binaryKeyMapType,
+ new GenericMap(singletonMap("k".getBytes(), null)),
+ new GenericMap(singletonMap("k".getBytes(), null)),
+ true);
+ }
+
private static Map<Object, Object> singletonMap(Object key, Object value) {
Map<Object, Object> map = new HashMap<>();
map.put(key, value);
