This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 140ad68dc13e fix(flink): preserve IN matches when literal list
contains NULL (#20517)
140ad68dc13e is described below
commit 140ad68dc13e128090cb11a5ce98f4b227bec35b
Author: Shuo Cheng <[email protected]>
AuthorDate: Fri Nov 6 14:43:10 2026 +0800
fix(flink): preserve IN matches when literal list contains NULL (#20517)
* fix(flink): preserve IN matches when literal list contains NULL
* refactor(flink): simplify all-null IN candidate check
---
.../apache/hudi/source/ExpressionEvaluators.java | 22 +++++-----
.../java/org/apache/hudi/util/ExpressionUtils.java | 8 ++--
.../hudi/source/TestExpressionEvaluators.java | 48 +++++++++++++++++-----
.../org/apache/hudi/util/TestExpressionUtils.java | 2 +
4 files changed, 56 insertions(+), 24 deletions(-)
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionEvaluators.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionEvaluators.java
index 311e5ddea832..81607226750b 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionEvaluators.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionEvaluators.java
@@ -104,15 +104,7 @@ public class ExpressionEvaluators {
.bindFieldReference(rExpr);
}
- boolean hasNullLiteral =
- childExprs.stream().anyMatch(e ->
- e instanceof ValueLiteralExpression
- &&
ExpressionUtils.getValueFromLiteral((ValueLiteralExpression) e) == null);
- if (hasNullLiteral) {
- return AlwaysFalse.getInstance();
- }
-
- // handle IN specifically
+ // handle IN specifically: NULL candidates do not prevent other values
from matching.
if (BuiltInFunctionDefinitions.IN.equals(funDef)) {
ValidationUtils.checkState(normalized, "The IN expression expects to be
normalized");
In in = In.getInstance();
@@ -122,6 +114,14 @@ public class ExpressionEvaluators {
return in;
}
+ boolean hasNullLiteral =
+ childExprs.stream().anyMatch(e ->
+ e instanceof ValueLiteralExpression
+ &&
ExpressionUtils.getValueFromLiteral((ValueLiteralExpression) e) == null);
+ if (hasNullLiteral) {
+ return AlwaysFalse.getInstance();
+ }
+
NullFalseEvaluator evaluator;
// handle binary operators
if (BuiltInFunctionDefinitions.EQUALS.equals(funDef)) {
@@ -409,7 +409,7 @@ public class ExpressionEvaluators {
@Override
public boolean eval(Map<String, ColumnStats> columnStatsMap) {
- if (Arrays.stream(vals).anyMatch(Objects::isNull)) {
+ if (Arrays.stream(vals).allMatch(Objects::isNull)) {
return false;
}
ColumnStats columnStats = getColumnStats(columnStatsMap);
@@ -425,7 +425,7 @@ public class ExpressionEvaluators {
return true;
}
- return Arrays.stream(vals).anyMatch(v ->
+ return Arrays.stream(vals).filter(Objects::nonNull).anyMatch(v ->
compare(minVal, v, this.type) <= 0 && compare(maxVal, v, this.type)
>= 0);
}
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ExpressionUtils.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ExpressionUtils.java
index 1783057beeb7..a70b8c63e6e9 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ExpressionUtils.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ExpressionUtils.java
@@ -142,12 +142,14 @@ public class ExpressionUtils {
/**
* Returns the value with given value literal expression.
*
- * <p>Returns null if the value can not parse as the output data type
correctly,
- * should call {@code ValueLiteralExpression.isNull} first to decide whether
- * the literal is NULL.
+ * <p>Returns null for NULL literals or if the value cannot be parsed as the
output data type.
+ * Use {@code ValueLiteralExpression.isNull} to distinguish a NULL literal
from a conversion failure.
*/
@Nullable
public static Object getValueFromLiteral(ValueLiteralExpression expr) {
+ if (expr.isNull()) {
+ return null;
+ }
LogicalType logicalType = expr.getOutputDataType().getLogicalType();
switch (logicalType.getTypeRoot()) {
case TIMESTAMP_WITHOUT_TIME_ZONE:
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java
index 4ae717306e22..339eb43ce6b2 100644
---
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java
@@ -367,14 +367,43 @@ public class TestExpressionEvaluators {
Map<String, ColumnStats> stats6 = convertColumnStats(indexRow6,
queryFields(2));
assertFalse(in.eval(stats6), "12 <> null");
+ in.bindVals(12, null);
+ assertTrue(in.eval(stats1), "A NULL candidate must not hide a matching
value");
+ assertFalse(in.eval(stats5), "A NULL candidate does not match non-null
values");
+ assertFalse(in.eval(stats6), "NULL column values do not match");
+
in.bindVals((Object) null);
- assertFalse(in.eval(stats1), "It is not possible to test for NULL values
with 'in' operator");
+ assertFalse(in.eval(stats1), "An all-NULL list cannot match");
+
+ Object[] largeList = new Object[201];
+ in.bindVals(largeList);
+ assertFalse(in.eval(stats1), "Even a large all-NULL list cannot match");
+ largeList[0] = 12;
+ assertTrue(in.eval(stats1), "A large list with a non-null value is
retained");
+ }
+
+ @Test
+ void testInExpressionWithNull() {
+ FieldReferenceExpression ref = new FieldReferenceExpression("f_int",
DataTypes.INT(), 2, 2);
+ ValueLiteralExpression twelve = new ValueLiteralExpression(12);
+ for (DataType nullType : Arrays.asList(DataTypes.INT(), DataTypes.NULL()))
{
+ ValueLiteralExpression nullLiteral = new ValueLiteralExpression(null,
nullType);
+ CallExpression expression = CallExpression.permanent(
+ BuiltInFunctionDefinitions.IN, Arrays.asList(ref, nullLiteral,
twelve), DataTypes.BOOLEAN());
+ ExpressionEvaluators.Evaluator evaluator = fromExpression(expression);
+
+ assertTrue(evaluator.eval(convertColumnStats(intIndexRow(11, 13),
queryFields(2))));
+ assertFalse(evaluator.eval(convertColumnStats(intIndexRow(13, 14),
queryFields(2))));
+ assertFalse(evaluator.eval(convertColumnStats(intIndexRow(null, null),
queryFields(2))));
+ CallExpression allNull = CallExpression.permanent(
+ BuiltInFunctionDefinitions.IN, Arrays.asList(ref, nullLiteral),
DataTypes.BOOLEAN());
+
assertFalse(fromExpression(allNull).eval(convertColumnStats(intIndexRow(11,
13), queryFields(2))));
+ }
}
@Test
void testAlwaysFalse() {
FieldReferenceExpression ref = new FieldReferenceExpression("f_int",
DataTypes.INT(), 2, 2);
- ValueLiteralExpression nullLiteral = new ValueLiteralExpression(null,
DataTypes.INT());
RowData indexRow = intIndexRow(11, 13);
Map<String, ColumnStats> stats = convertColumnStats(indexRow,
queryFields(2));
BuiltInFunctionDefinition[] funDefs = new BuiltInFunctionDefinition[] {
@@ -385,14 +414,13 @@ public class TestExpressionEvaluators {
BuiltInFunctionDefinitions.LESS_THAN_OR_EQUAL,
BuiltInFunctionDefinitions.GREATER_THAN_OR_EQUAL,
BuiltInFunctionDefinitions.IN};
- for (BuiltInFunctionDefinition funDef : funDefs) {
- CallExpression expr =
- CallExpression.permanent(
- funDef,
- Arrays.asList(ref, nullLiteral),
- DataTypes.BOOLEAN());
- // always return false if the literal value is null
- assertFalse(fromExpression(expr).eval(stats));
+ for (DataType nullType : Arrays.asList(DataTypes.INT(), DataTypes.NULL()))
{
+ ValueLiteralExpression nullLiteral = new ValueLiteralExpression(null,
nullType);
+ for (BuiltInFunctionDefinition funDef : funDefs) {
+ CallExpression expr = CallExpression.permanent(
+ funDef, Arrays.asList(ref, nullLiteral), DataTypes.BOOLEAN());
+ assertFalse(fromExpression(expr).eval(stats));
+ }
}
}
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/util/TestExpressionUtils.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/util/TestExpressionUtils.java
index d2d733ba80c3..ac29a841e99c 100644
---
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/util/TestExpressionUtils.java
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/util/TestExpressionUtils.java
@@ -41,6 +41,7 @@ import java.util.List;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
class TestExpressionUtils {
@@ -79,6 +80,7 @@ class TestExpressionUtils {
@Test
void getValueFromLiteralForNull() {
+ assertNull(ExpressionUtils.getValueFromLiteral(new
ValueLiteralExpression(null, DataTypes.NULL())));
List<RowType.RowField> fields = ((RowType)
ROW_DATA_TYPE.getLogicalType()).getFields();
List<DataType> dataTypes = ROW_DATA_TYPE.getChildren();
CallExpression callExpression;