This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 140ad68dc13e fix(flink): preserve IN matches when literal list 
contains NULL (#20517)
140ad68dc13e is described below

commit 140ad68dc13e128090cb11a5ce98f4b227bec35b
Author: Shuo Cheng <[email protected]>
AuthorDate: Fri Nov 6 14:43:10 2026 +0800

    fix(flink): preserve IN matches when literal list contains NULL (#20517)
    
    * fix(flink): preserve IN matches when literal list contains NULL
    
    * refactor(flink): simplify all-null IN candidate check
---
 .../apache/hudi/source/ExpressionEvaluators.java   | 22 +++++-----
 .../java/org/apache/hudi/util/ExpressionUtils.java |  8 ++--
 .../hudi/source/TestExpressionEvaluators.java      | 48 +++++++++++++++++-----
 .../org/apache/hudi/util/TestExpressionUtils.java  |  2 +
 4 files changed, 56 insertions(+), 24 deletions(-)

diff --git 
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionEvaluators.java
 
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionEvaluators.java
index 311e5ddea832..81607226750b 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionEvaluators.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionEvaluators.java
@@ -104,15 +104,7 @@ public class ExpressionEvaluators {
           .bindFieldReference(rExpr);
     }
 
-    boolean hasNullLiteral =
-        childExprs.stream().anyMatch(e ->
-            e instanceof ValueLiteralExpression
-                && 
ExpressionUtils.getValueFromLiteral((ValueLiteralExpression) e) == null);
-    if (hasNullLiteral) {
-      return AlwaysFalse.getInstance();
-    }
-
-    // handle IN specifically
+    // handle IN specifically: NULL candidates do not prevent other values 
from matching.
     if (BuiltInFunctionDefinitions.IN.equals(funDef)) {
       ValidationUtils.checkState(normalized, "The IN expression expects to be 
normalized");
       In in = In.getInstance();
@@ -122,6 +114,14 @@ public class ExpressionEvaluators {
       return in;
     }
 
+    boolean hasNullLiteral =
+        childExprs.stream().anyMatch(e ->
+            e instanceof ValueLiteralExpression
+                && 
ExpressionUtils.getValueFromLiteral((ValueLiteralExpression) e) == null);
+    if (hasNullLiteral) {
+      return AlwaysFalse.getInstance();
+    }
+
     NullFalseEvaluator evaluator;
     // handle binary operators
     if (BuiltInFunctionDefinitions.EQUALS.equals(funDef)) {
@@ -409,7 +409,7 @@ public class ExpressionEvaluators {
 
     @Override
     public boolean eval(Map<String, ColumnStats> columnStatsMap) {
-      if (Arrays.stream(vals).anyMatch(Objects::isNull)) {
+      if (Arrays.stream(vals).allMatch(Objects::isNull)) {
         return false;
       }
       ColumnStats columnStats = getColumnStats(columnStatsMap);
@@ -425,7 +425,7 @@ public class ExpressionEvaluators {
         return true;
       }
 
-      return Arrays.stream(vals).anyMatch(v ->
+      return Arrays.stream(vals).filter(Objects::nonNull).anyMatch(v ->
           compare(minVal, v, this.type) <= 0 && compare(maxVal, v, this.type) 
>= 0);
     }
 
diff --git 
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ExpressionUtils.java
 
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ExpressionUtils.java
index 1783057beeb7..a70b8c63e6e9 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ExpressionUtils.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ExpressionUtils.java
@@ -142,12 +142,14 @@ public class ExpressionUtils {
   /**
    * Returns the value with given value literal expression.
    *
-   * <p>Returns null if the value can not parse as the output data type 
correctly,
-   * should call {@code ValueLiteralExpression.isNull} first to decide whether
-   * the literal is NULL.
+   * <p>Returns null for NULL literals or if the value cannot be parsed as the 
output data type.
+   * Use {@code ValueLiteralExpression.isNull} to distinguish a NULL literal 
from a conversion failure.
    */
   @Nullable
   public static Object getValueFromLiteral(ValueLiteralExpression expr) {
+    if (expr.isNull()) {
+      return null;
+    }
     LogicalType logicalType = expr.getOutputDataType().getLogicalType();
     switch (logicalType.getTypeRoot()) {
       case TIMESTAMP_WITHOUT_TIME_ZONE:
diff --git 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java
 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java
index 4ae717306e22..339eb43ce6b2 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java
@@ -367,14 +367,43 @@ public class TestExpressionEvaluators {
     Map<String, ColumnStats> stats6 = convertColumnStats(indexRow6, 
queryFields(2));
     assertFalse(in.eval(stats6), "12 <> null");
 
+    in.bindVals(12, null);
+    assertTrue(in.eval(stats1), "A NULL candidate must not hide a matching 
value");
+    assertFalse(in.eval(stats5), "A NULL candidate does not match non-null 
values");
+    assertFalse(in.eval(stats6), "NULL column values do not match");
+
     in.bindVals((Object) null);
-    assertFalse(in.eval(stats1), "It is not possible to test for NULL values 
with 'in' operator");
+    assertFalse(in.eval(stats1), "An all-NULL list cannot match");
+
+    Object[] largeList = new Object[201];
+    in.bindVals(largeList);
+    assertFalse(in.eval(stats1), "Even a large all-NULL list cannot match");
+    largeList[0] = 12;
+    assertTrue(in.eval(stats1), "A large list with a non-null value is 
retained");
+  }
+
+  @Test
+  void testInExpressionWithNull() {
+    FieldReferenceExpression ref = new FieldReferenceExpression("f_int", 
DataTypes.INT(), 2, 2);
+    ValueLiteralExpression twelve = new ValueLiteralExpression(12);
+    for (DataType nullType : Arrays.asList(DataTypes.INT(), DataTypes.NULL())) 
{
+      ValueLiteralExpression nullLiteral = new ValueLiteralExpression(null, 
nullType);
+      CallExpression expression = CallExpression.permanent(
+          BuiltInFunctionDefinitions.IN, Arrays.asList(ref, nullLiteral, 
twelve), DataTypes.BOOLEAN());
+      ExpressionEvaluators.Evaluator evaluator = fromExpression(expression);
+
+      assertTrue(evaluator.eval(convertColumnStats(intIndexRow(11, 13), 
queryFields(2))));
+      assertFalse(evaluator.eval(convertColumnStats(intIndexRow(13, 14), 
queryFields(2))));
+      assertFalse(evaluator.eval(convertColumnStats(intIndexRow(null, null), 
queryFields(2))));
+      CallExpression allNull = CallExpression.permanent(
+          BuiltInFunctionDefinitions.IN, Arrays.asList(ref, nullLiteral), 
DataTypes.BOOLEAN());
+      
assertFalse(fromExpression(allNull).eval(convertColumnStats(intIndexRow(11, 
13), queryFields(2))));
+    }
   }
 
   @Test
   void testAlwaysFalse() {
     FieldReferenceExpression ref = new FieldReferenceExpression("f_int", 
DataTypes.INT(), 2, 2);
-    ValueLiteralExpression nullLiteral = new ValueLiteralExpression(null, 
DataTypes.INT());
     RowData indexRow = intIndexRow(11, 13);
     Map<String, ColumnStats> stats = convertColumnStats(indexRow, 
queryFields(2));
     BuiltInFunctionDefinition[] funDefs = new BuiltInFunctionDefinition[] {
@@ -385,14 +414,13 @@ public class TestExpressionEvaluators {
         BuiltInFunctionDefinitions.LESS_THAN_OR_EQUAL,
         BuiltInFunctionDefinitions.GREATER_THAN_OR_EQUAL,
         BuiltInFunctionDefinitions.IN};
-    for (BuiltInFunctionDefinition funDef : funDefs) {
-      CallExpression expr =
-          CallExpression.permanent(
-              funDef,
-              Arrays.asList(ref, nullLiteral),
-              DataTypes.BOOLEAN());
-      // always return false if the literal value is null
-      assertFalse(fromExpression(expr).eval(stats));
+    for (DataType nullType : Arrays.asList(DataTypes.INT(), DataTypes.NULL())) 
{
+      ValueLiteralExpression nullLiteral = new ValueLiteralExpression(null, 
nullType);
+      for (BuiltInFunctionDefinition funDef : funDefs) {
+        CallExpression expr = CallExpression.permanent(
+            funDef, Arrays.asList(ref, nullLiteral), DataTypes.BOOLEAN());
+        assertFalse(fromExpression(expr).eval(stats));
+      }
     }
   }
 
diff --git 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/util/TestExpressionUtils.java
 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/util/TestExpressionUtils.java
index d2d733ba80c3..ac29a841e99c 100644
--- 
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/util/TestExpressionUtils.java
+++ 
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/util/TestExpressionUtils.java
@@ -41,6 +41,7 @@ import java.util.List;
 
 import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 class TestExpressionUtils {
@@ -79,6 +80,7 @@ class TestExpressionUtils {
   
   @Test
   void getValueFromLiteralForNull() {
+    assertNull(ExpressionUtils.getValueFromLiteral(new 
ValueLiteralExpression(null, DataTypes.NULL())));
     List<RowType.RowField> fields = ((RowType) 
ROW_DATA_TYPE.getLogicalType()).getFields();
     List<DataType> dataTypes = ROW_DATA_TYPE.getChildren();
     CallExpression callExpression;

Reply via email to