This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new e3bae975d8d [fix](fe) Preserve Paimon CAST predicate semantics on
branch-4.1 (#67933)
e3bae975d8d is described below
commit e3bae975d8dccb3ba0f1700219e7eb822a8159e2
Author: Gabriel <[email protected]>
AuthorDate: Tue Sep 15 09:24:35 2026 +0800
[fix](fe) Preserve Paimon CAST predicate semantics on branch-4.1 (#67933)
### What problem does this PR solve?
Problem Summary: Paimon predicate conversion on branch-4.1 strips CAST
from column references before building source filters. For a STRING
column containing '05', ' 5', and '5', `CAST(code AS INT) = 5` can
become the source predicate `code = '5'`, incorrectly pruning matching
rows before Doris evaluates the original predicate. Decimal casts that
reduce scale have the same risk.
Only convert bare column references. Keep CAST predicates in Doris,
preserving their value and null semantics. Ordinary column predicates
can still be pushed down independently. This follows the conservative
CAST policy already used by the connector on master.
### Release note
Fix missing rows in Paimon queries with predicates on casted columns on
branch-4.1.
### Check List (For Author)
- Test: FE planner regression tests for CAST comparisons, IN/NOT IN,
null checks, OR, independent AND conjuncts, and ordinary column
pushdown. The 11 predicate-converter tests pass after reproducing 8
failures on the original code. FE Checkstyle passes with 0 violations.
All 59 tests across PaimonPredicateConverterTest, PaimonScanNodeTest,
and PaimonSourceTest pass. No external Paimon cluster was required for
these FE tests.
- Behavior changed: Yes. CAST column predicates remain in Doris instead
of being rewritten into potentially stricter Paimon filters. This may
reduce pushdown for otherwise safe casts.
- Does this need documentation: No.
---
.../paimon/source/PaimonPredicateConverter.java | 11 ++---
.../planner/PaimonPredicateConverterTest.java | 56 ++++++++++++++++++++++
2 files changed, 60 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java
index ae45c242718..45b351aa618 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java
@@ -185,15 +185,12 @@ public class PaimonPredicateConverter {
public static SlotRef convertDorisExprToSlotRef(Expr expr) {
- SlotRef slotRef = null;
+ // Stripping CAST can prune matching rows, e.g. CAST('05' AS INT) = 5
is not '05' = '5'.
+ // Keep casted columns in the original Doris conjuncts to preserve
value and null semantics.
if (expr instanceof SlotRef) {
- slotRef = (SlotRef) expr;
- } else if (expr instanceof CastExpr) {
- if (expr.getChild(0) instanceof SlotRef) {
- slotRef = (SlotRef) expr.getChild(0);
- }
+ return (SlotRef) expr;
}
- return slotRef;
+ return null;
}
public LiteralExpr convertDorisExprToLiteralExpr(Expr expr) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java
index fde1b6f74c2..85ff507ddb6 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.planner;
+import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.Expr;
import org.apache.doris.common.FeConstants;
import org.apache.doris.datasource.paimon.source.PaimonPredicateConverter;
@@ -29,10 +30,14 @@ import org.apache.paimon.predicate.LeafPredicate;
import org.apache.paimon.predicate.Or;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.VarCharType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.util.List;
@@ -46,6 +51,57 @@ public class PaimonPredicateConverterTest extends
TestWithFeService {
String tbl1 = "create table db1.tbl1(" + "k1 int," + " k2 int," + " v1
int)" + " distributed by hash(k1)"
+ " properties('replication_num' = '1');";
createTables(tbl1);
+ createTables("create table db1.cast_predicates (id int, code string,
amount decimal(10, 2))"
+ + " distributed by hash(id) properties('replication_num' =
'1')");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"cast(code as int) = 5", "5 = cast(code as int)",
"cast(code as int) > 5",
+ "cast(code as int) in (5, 6)", "cast(code as int) not in (5, 6)",
+ "cast(code as int) is null", "cast(code as int) is not null",
+ "cast(code as int) = 5 or id = 1"})
+ public void rejectStringToIntegerCast(String predicate) throws Exception {
+ List<Expr> conjuncts = planCastPredicate(predicate);
+ Assertions.assertTrue(conjuncts.stream().anyMatch(expr ->
expr.contains(CastExpr.class)), predicate);
+ List<Expr> remaining = Expr.cloneList(conjuncts);
+
Assertions.assertTrue(castPredicateConverter().convertToPaimonExpr(conjuncts).isEmpty(),
predicate);
+ Assertions.assertEquals(remaining, conjuncts);
+ }
+
+ @Test
+ public void rejectDecimalScaleCast() throws Exception {
+ List<Expr> conjuncts = planCastPredicate("cast(amount as decimal(10,
1)) = 1.2");
+ Assertions.assertTrue(conjuncts.stream().anyMatch(expr ->
expr.contains(CastExpr.class)));
+
Assertions.assertTrue(castPredicateConverter().convertToPaimonExpr(conjuncts).isEmpty());
+ }
+
+ @Test
+ public void retainIndependentUncastPredicates() throws Exception {
+ List<Expr> conjuncts = planCastPredicate("cast(code as int) = 5 and id
= 1");
+ List<Expr> remaining = Expr.cloneList(conjuncts);
+ List<Predicate> predicates =
castPredicateConverter().convertToPaimonExpr(conjuncts);
+ Assertions.assertEquals(1, predicates.size());
+ Assertions.assertEquals("id", ((LeafPredicate)
predicates.get(0)).fieldName());
+ Assertions.assertEquals(remaining, conjuncts);
+
+ for (String predicate : Lists.newArrayList("code = '5'", "id > 1", "id
in (1, 2)")) {
+ Assertions.assertEquals(1,
castPredicateConverter().convertToPaimonExpr(planCastPredicate(predicate)).size(),
+ predicate);
+ }
+ }
+
+ private List<Expr> planCastPredicate(String predicate) throws Exception {
+ StmtExecutor executor = new StmtExecutor(connectContext,
+ "select * from db1.cast_predicates where " + predicate);
+ executor.execute();
+ return executor.planner().getScanNodes().get(0).getConjuncts();
+ }
+
+ private PaimonPredicateConverter castPredicateConverter() {
+ return new PaimonPredicateConverter(new RowType(Lists.newArrayList(
+ new DataField(0, "id", new IntType()),
+ new DataField(1, "code", new VarCharType()),
+ new DataField(2, "amount", new DecimalType(10, 2)))));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
