This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 11ae6ef981 [core] Optimize between in Between
11ae6ef981 is described below
commit 11ae6ef9818c154a6c5556d3aea2ba7e7893758c
Author: JingsongLi <[email protected]>
AuthorDate: Fri Feb 13 16:31:36 2026 +0800
[core] Optimize between in Between
---
.../bitmap/RangeBitmapIndexBenchmark.java | 4 +-
.../apache/paimon/fileindex/FileIndexReader.java | 10 +
.../bsi/BitSliceIndexBitmapFileIndex.java | 17 +-
.../bsi/BitSliceIndexBitmapFileIndexFactory.java | 2 +-
.../rangebitmap/RangeBitmapFileIndex.java | 10 +
.../java/org/apache/paimon/predicate/Between.java | 166 ++++++++++
.../apache/paimon/predicate/PredicateBuilder.java | 6 +-
.../org/apache/paimon/utils/PredicateUtils.java | 190 -----------
.../bsi/BitSliceIndexBitmapFileIndexTest.java | 6 +-
.../org/apache/paimon/predicate/BetweenTest.java | 364 +++++++++++++++++++++
.../paimon/predicate/PredicateBuilderTest.java | 6 +-
.../apache/paimon/utils/PredicateUtilsTest.java | 207 ------------
.../paimon/table/source/ReadBuilderImpl.java | 2 -
13 files changed, 577 insertions(+), 413 deletions(-)
diff --git
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/bitmap/RangeBitmapIndexBenchmark.java
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/bitmap/RangeBitmapIndexBenchmark.java
index 6787f8dabf..11b0a5d088 100644
---
a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/bitmap/RangeBitmapIndexBenchmark.java
+++
b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/bitmap/RangeBitmapIndexBenchmark.java
@@ -166,7 +166,7 @@ public class RangeBitmapIndexBenchmark {
Random random = new Random();
FileIndexWriter bsiWriter =
- new BitSliceIndexBitmapFileIndex(DataTypes.INT(), new
Options()).createWriter();
+ new
BitSliceIndexBitmapFileIndex(DataTypes.INT()).createWriter();
Options bitmapOptions = new Options();
bitmapOptions.setInteger(BitmapFileIndex.VERSION,
BitmapFileIndex.VERSION_2);
@@ -219,7 +219,7 @@ public class RangeBitmapIndexBenchmark {
LocalFileIO.LocalSeekableInputStream localSeekableInputStream =
new LocalFileIO.LocalSeekableInputStream(file);
FileIndexReader reader =
- new BitSliceIndexBitmapFileIndex(DataTypes.INT(), options)
+ new BitSliceIndexBitmapFileIndex(DataTypes.INT())
.createReader(localSeekableInputStream, 0, 0);
FileIndexResult result = function.apply(reader, fieldRef);
((BitmapIndexResult) result).get();
diff --git
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexReader.java
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexReader.java
index 3b73e62d2b..cfb460f36e 100644
---
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexReader.java
+++
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexReader.java
@@ -135,4 +135,14 @@ public abstract class FileIndexReader implements
FunctionVisitor<FileIndexResult
public FileIndexResult visitNonFieldLeaf(LeafPredicate predicate) {
return REMAIN;
}
+
+ @Override
+ public FileIndexResult visitBetween(FieldRef fieldRef, Object from, Object
to) {
+ return REMAIN;
+ }
+
+ @Override
+ public FileIndexResult visitNotBetween(FieldRef fieldRef, Object from,
Object to) {
+ return REMAIN;
+ }
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java
b/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java
index df6b8d897c..680008a736 100644
---
a/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java
+++
b/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java
@@ -26,7 +26,6 @@ import org.apache.paimon.fileindex.FileIndexWriter;
import org.apache.paimon.fileindex.FileIndexer;
import org.apache.paimon.fileindex.bitmap.BitmapIndexResult;
import org.apache.paimon.fs.SeekableInputStream;
-import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.FieldRef;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.DataType;
@@ -59,7 +58,7 @@ public class BitSliceIndexBitmapFileIndex implements
FileIndexer {
private final DataType dataType;
- public BitSliceIndexBitmapFileIndex(DataType dataType, Options options) {
+ public BitSliceIndexBitmapFileIndex(DataType dataType) {
this.dataType = dataType;
}
@@ -290,7 +289,7 @@ public class BitSliceIndexBitmapFileIndex implements
FileIndexer {
}
@Override
- public FileIndexResult visitLessOrEqual(FieldRef fieldRef, Object
literal) {
+ public BitmapIndexResult visitLessOrEqual(FieldRef fieldRef, Object
literal) {
return new BitmapIndexResult(
() -> {
Long value = valueMapper.apply(literal);
@@ -317,7 +316,7 @@ public class BitSliceIndexBitmapFileIndex implements
FileIndexer {
}
@Override
- public FileIndexResult visitGreaterOrEqual(FieldRef fieldRef, Object
literal) {
+ public BitmapIndexResult visitGreaterOrEqual(FieldRef fieldRef, Object
literal) {
return new BitmapIndexResult(
() -> {
Long value = valueMapper.apply(literal);
@@ -329,6 +328,16 @@ public class BitSliceIndexBitmapFileIndex implements
FileIndexer {
}
});
}
+
+ @Override
+ public FileIndexResult visitBetween(FieldRef fieldRef, Object from,
Object to) {
+ return new BitmapIndexResult(
+ () -> {
+ RoaringBitmap32 gte = visitGreaterOrEqual(fieldRef,
from).get();
+ RoaringBitmap32 lte = visitLessOrEqual(fieldRef,
to).get();
+ return RoaringBitmap32.and(gte, lte);
+ });
+ }
}
public static Function<Object, Long> getValueMapper(DataType dataType) {
diff --git
a/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexFactory.java
b/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexFactory.java
index aa7a92b785..c6d182ce01 100644
---
a/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexFactory.java
+++
b/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexFactory.java
@@ -35,6 +35,6 @@ public class BitSliceIndexBitmapFileIndexFactory implements
FileIndexerFactory {
@Override
public FileIndexer create(DataType dataType, Options options) {
- return new BitSliceIndexBitmapFileIndex(dataType, options);
+ return new BitSliceIndexBitmapFileIndex(dataType);
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java
b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java
index 1482e24f13..8b3ef92f22 100644
---
a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java
+++
b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java
@@ -153,6 +153,16 @@ public class RangeBitmapFileIndex implements FileIndexer {
return new BitmapIndexResult(() ->
bitmap.gte(converter.apply(literal)));
}
+ @Override
+ public FileIndexResult visitBetween(FieldRef fieldRef, Object from,
Object to) {
+ return new BitmapIndexResult(
+ () -> {
+ RoaringBitmap32 gte =
bitmap.gte(converter.apply(from));
+ RoaringBitmap32 lte = bitmap.lte(converter.apply(to));
+ return RoaringBitmap32.and(gte, lte);
+ });
+ }
+
@Override
public FileIndexResult visitTopN(TopN topN, FileIndexResult result) {
RoaringBitmap32 foundSet =
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/Between.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/Between.java
index c9124c14b4..30d317b224 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/Between.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/Between.java
@@ -22,10 +22,15 @@ import org.apache.paimon.types.DataType;
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import static org.apache.paimon.predicate.CompareUtils.compareLiteral;
+import static org.apache.paimon.predicate.PredicateBuilder.splitAnd;
/** The {@link LeafFunction} to eval between. */
public class Between extends LeafTernaryFunction {
@@ -72,4 +77,165 @@ public class Between extends LeafTernaryFunction {
public String toJson() {
return NAME;
}
+
+ /**
+ * Optimize predicates by converting LessOrEqual and GreaterOrEqual to
Between for the same
+ * field.
+ */
+ public static List<Predicate> optimize(List<Predicate> predicates) {
+ // First, split all AND predicates to get leaf predicates
+ List<Predicate> leafPredicates = new ArrayList<>();
+ for (Predicate predicate : predicates) {
+ leafPredicates.addAll(splitAnd(predicate));
+ }
+
+ // Group leaf predicates by their FieldTransform
+ Map<FieldTransform, List<LeafPredicate>> predicatesByField = new
HashMap<>();
+ List<Predicate> otherPredicates = new ArrayList<>();
+
+ for (Predicate predicate : leafPredicates) {
+ if (predicate instanceof LeafPredicate) {
+ LeafPredicate leafPredicate = (LeafPredicate) predicate;
+ if (leafPredicate.transform() instanceof FieldTransform) {
+ FieldTransform fieldTransform = (FieldTransform)
leafPredicate.transform();
+ predicatesByField
+ .computeIfAbsent(fieldTransform, k -> new
ArrayList<>())
+ .add(leafPredicate);
+ } else {
+ otherPredicates.add(predicate);
+ }
+ } else {
+ otherPredicates.add(predicate);
+ }
+ }
+
+ List<Predicate> result = new ArrayList<>(otherPredicates);
+ for (Map.Entry<FieldTransform, List<LeafPredicate>> entry :
predicatesByField.entrySet()) {
+ FieldTransform field = entry.getKey();
+ List<LeafPredicate> fieldPredicates = entry.getValue();
+ fieldPredicates = mergeLessAndGreaterToBetween(field,
fieldPredicates);
+ fieldPredicates = mergeMultipleBetweens(field, fieldPredicates);
+ result.addAll(fieldPredicates);
+ }
+
+ // return input predicate if nothing changed
+ return result.size() == predicates.size() ? predicates : result;
+ }
+
+ private static List<LeafPredicate> mergeLessAndGreaterToBetween(
+ FieldTransform field, List<LeafPredicate> predicates) {
+ List<LeafPredicate> result = new ArrayList<>();
+ DataType type = field.outputType();
+ LeafPredicate lessOrEqual = null;
+ LeafPredicate greaterOrEqual = null;
+
+ for (LeafPredicate leafPredicate : predicates) {
+ if (leafPredicate.function() == LessOrEqual.INSTANCE) {
+ if (lessOrEqual == null) {
+ lessOrEqual = leafPredicate;
+ } else {
+ lessOrEqual =
+ compareLiteral(
+ type,
+
lessOrEqual.literals().get(0),
+
leafPredicate.literals().get(0))
+ < 0
+ ? lessOrEqual
+ : leafPredicate;
+ }
+ } else if (leafPredicate.function() == GreaterOrEqual.INSTANCE) {
+ if (greaterOrEqual == null) {
+ greaterOrEqual = leafPredicate;
+ } else {
+ greaterOrEqual =
+ compareLiteral(
+ type,
+
greaterOrEqual.literals().get(0),
+
leafPredicate.literals().get(0))
+ > 0
+ ? greaterOrEqual
+ : leafPredicate;
+ }
+ } else {
+ result.add(leafPredicate);
+ }
+ }
+
+ // If we have both LessOrEqual and GreaterOrEqual, convert to Between
+ if (lessOrEqual != null && greaterOrEqual != null) {
+ // Determine which is the lower bound and which is the upper bound
+ Object lowerBound = greaterOrEqual.literals().get(0);
+ Object upperBound = lessOrEqual.literals().get(0);
+ if (compareLiteral(type, lowerBound, upperBound) >= 0) {
+ // No valid intersection, keep all original predicates
+ result.add(lessOrEqual);
+ result.add(greaterOrEqual);
+ } else {
+ // convert to Between
+ LeafPredicate betweenPredicate =
+ LeafPredicate.of(
+ field, Between.INSTANCE,
Arrays.asList(lowerBound, upperBound));
+ result.add(betweenPredicate);
+ }
+ } else {
+ // Add LessOrEqual and GreaterOrEqual to remaining if no pair found
+ if (lessOrEqual != null) {
+ result.add(lessOrEqual);
+ }
+ if (greaterOrEqual != null) {
+ result.add(greaterOrEqual);
+ }
+ }
+
+ return result;
+ }
+
+ private static List<LeafPredicate> mergeMultipleBetweens(
+ FieldTransform field, List<LeafPredicate> predicates) {
+ List<LeafPredicate> results = new ArrayList<>();
+ List<LeafPredicate> betweens = new ArrayList<>();
+ for (LeafPredicate predicate : predicates) {
+ if (predicate.function() == Between.INSTANCE) {
+ betweens.add(predicate);
+ } else {
+ results.add(predicate);
+ }
+ }
+ if (betweens.isEmpty() || betweens.size() == 1) {
+ return predicates;
+ }
+
+ DataType fieldType = field.outputType();
+
+ // Find the maximum lower bound and minimum upper bound
+ Object maxLower = null;
+ Object minUpper = null;
+
+ for (LeafPredicate between : betweens) {
+ Object lower = between.literals().get(0);
+ Object upper = between.literals().get(1);
+
+ if (maxLower == null || compareLiteral(fieldType, lower, maxLower)
> 0) {
+ maxLower = lower;
+ }
+ if (minUpper == null || compareLiteral(fieldType, upper, minUpper)
< 0) {
+ minUpper = upper;
+ }
+ }
+
+ // Check if intersection is valid
+ if (maxLower != null
+ && minUpper != null
+ && compareLiteral(fieldType, maxLower, minUpper) <= 0) {
+ // Valid intersection, create a single Between predicate
+ LeafPredicate mergedBetween =
+ LeafPredicate.of(field, Between.INSTANCE,
Arrays.asList(maxLower, minUpper));
+ results.add(mergedBetween);
+ } else {
+ // No valid intersection, keep all original predicates unchanged
+ results.addAll(betweens);
+ }
+
+ return results;
+ }
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateBuilder.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateBuilder.java
index c25764193d..79c26fc351 100644
---
a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateBuilder.java
+++
b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateBuilder.java
@@ -254,7 +254,11 @@ public class PredicateBuilder {
if (predicates.size() == 1) {
return predicates.get(0);
}
- return predicates.stream()
+
+ // Optimize by converting LessOrEqual and GreaterOrEqual to Between
for same field
+ List<Predicate> optimized = Between.optimize(predicates);
+
+ return optimized.stream()
.reduce((a, b) -> new CompoundPredicate(And.INSTANCE,
Arrays.asList(a, b)))
.get();
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/utils/PredicateUtils.java
b/paimon-common/src/main/java/org/apache/paimon/utils/PredicateUtils.java
deleted file mode 100644
index 49964185f4..0000000000
--- a/paimon-common/src/main/java/org/apache/paimon/utils/PredicateUtils.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.utils;
-
-import org.apache.paimon.predicate.Between;
-import org.apache.paimon.predicate.CompareUtils;
-import org.apache.paimon.predicate.CompoundPredicate;
-import org.apache.paimon.predicate.GreaterOrEqual;
-import org.apache.paimon.predicate.LeafPredicate;
-import org.apache.paimon.predicate.LessOrEqual;
-import org.apache.paimon.predicate.Or;
-import org.apache.paimon.predicate.Predicate;
-import org.apache.paimon.predicate.PredicateBuilder;
-import org.apache.paimon.predicate.Transform;
-
-import javax.annotation.Nullable;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-
-/** Utils for {@link Predicate}. */
-public class PredicateUtils {
-
- /**
- * Try to rewrite possible {@code GREATER_OR_EQUAL} and {@code
LESS_OR_EQUAL} predicates to
- * {@code BETWEEN} leaf predicate. This method will recursively try to
rewrite the children
- * predicates of an {@code AND}, for example: {@code OR(a >= 1, AND(b >=
1, b <= 2))} will be
- * rewritten to {@code OR(a >= 1, BETWEEN(b, 1, 2))}.
- */
- public static Predicate tryRewriteBetweenPredicate(@Nullable Predicate
filter) {
- if (filter == null || filter instanceof LeafPredicate) {
- return filter;
- }
- CompoundPredicate compoundPredicate = (CompoundPredicate) filter;
- boolean isOr = compoundPredicate.function() instanceof Or;
-
- Map<Transform, List<LeafPredicate>> leavesByTransform = new
HashMap<>();
- List<Predicate> resultChildren = new ArrayList<>();
- // Flatten the children predicates of an AND
- // For example, for AND(b >= 1, AND(a >= 1, b <= 2)), we will get [a
>= 1, b >= 1, b <= 2]
- // After flattening, all children will be either LeafPredicate or
ORPredicate
- List<Predicate> effectiveChildren =
- isOr ? compoundPredicate.children() :
flattenChildren(compoundPredicate.children());
- for (Predicate child : effectiveChildren) {
- if (child instanceof LeafPredicate) {
- leavesByTransform
- .computeIfAbsent(
- ((LeafPredicate) child).transform(), k -> new
ArrayList<>())
- .add((LeafPredicate) child);
- } else {
- resultChildren.add(tryRewriteBetweenPredicate(child));
- }
- }
-
- for (Map.Entry<Transform, List<LeafPredicate>> leaves :
leavesByTransform.entrySet()) {
- if (isOr) {
- resultChildren.addAll(leaves.getValue());
- continue;
- }
-
- Transform transform = leaves.getKey();
-
- // for children predicates of an AND, we only need to reserve
- // the largest GREATER_OR_EQUAL and the smallest LESS_OR_EQUAL
- // For example, for AND(a >= 1, a >= 2, a <= 3, a <= 4), we only
need to reserve a >= 2
- // and a <= 3
- LeafPredicate gtePredicate = null;
- LeafPredicate ltePredicate = null;
- for (LeafPredicate leaf : leaves.getValue()) {
- if (leaf.function() instanceof GreaterOrEqual) {
- if (gtePredicate == null
- || CompareUtils.compareLiteral(
- transform.outputType(),
- leaf.literals().get(0),
- gtePredicate.literals().get(0))
- > 0) {
- gtePredicate = leaf;
- }
- } else if (leaf.function() instanceof LessOrEqual) {
- if (ltePredicate == null
- || CompareUtils.compareLiteral(
- transform.outputType(),
- leaf.literals().get(0),
- ltePredicate.literals().get(0))
- < 0) {
- ltePredicate = leaf;
- }
- } else {
- resultChildren.add(leaf);
- }
- }
-
- boolean converted = false;
- if (gtePredicate != null && ltePredicate != null) {
- Optional<Predicate> betweenLeaf =
convertToBetweenLeaf(gtePredicate, ltePredicate);
- if (betweenLeaf.isPresent()) {
- converted = true;
- resultChildren.add(betweenLeaf.get());
- }
- }
- if (!converted) {
- if (gtePredicate != null) {
- resultChildren.add(gtePredicate);
- }
- if (ltePredicate != null) {
- resultChildren.add(ltePredicate);
- }
- }
- }
-
- return isOr ? PredicateBuilder.or(resultChildren) :
PredicateBuilder.and(resultChildren);
- }
-
- private static List<Predicate> flattenChildren(List<Predicate> children) {
- List<Predicate> result = new ArrayList<>();
- for (Predicate child : children) {
- if (child instanceof LeafPredicate) {
- result.add(child);
- } else {
- CompoundPredicate compoundPredicate = (CompoundPredicate)
child;
- if (compoundPredicate.function() instanceof Or) {
- result.add(child);
- } else {
-
result.addAll(flattenChildren(compoundPredicate.children()));
- }
- }
- }
- return result;
- }
-
- /**
- * Convert child predicates of an AND to a BETWEEN leaf predicate. Return
`Optional.empty()` if
- * not possible.
- */
- public static Optional<Predicate> convertToBetweenLeaf(
- Predicate leftChild, Predicate rightChild) {
- if (leftChild instanceof LeafPredicate && rightChild instanceof
LeafPredicate) {
- LeafPredicate left = (LeafPredicate) leftChild;
- LeafPredicate right = (LeafPredicate) rightChild;
- if (Objects.equals(left.transform(), right.transform())) {
- if (left.function() instanceof GreaterOrEqual
- && right.function() instanceof LessOrEqual) {
- return createBetweenLeaf(left, right);
- } else if (left.function() instanceof LessOrEqual
- && right.function() instanceof GreaterOrEqual) {
- return createBetweenLeaf(right, left);
- }
- }
- }
-
- return Optional.empty();
- }
-
- private static Optional<Predicate> createBetweenLeaf(
- LeafPredicate gtePredicate, LeafPredicate ltePredicate) {
- // gtePredicate and ltePredicate should have the same transform
- Transform transform = gtePredicate.transform();
- Object lbLiteral = gtePredicate.literals().get(0);
- Object ubLiteral = ltePredicate.literals().get(0);
-
- if (CompareUtils.compareLiteral(transform.outputType(), lbLiteral,
ubLiteral) > 0) {
- return Optional.empty();
- }
-
- return Optional.of(
- new LeafPredicate(
- transform, Between.INSTANCE, Arrays.asList(lbLiteral,
ubLiteral)));
- }
-}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java
b/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java
index cd73f2b2a2..b55e2e77e1 100644
---
a/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java
@@ -39,7 +39,7 @@ public class BitSliceIndexBitmapFileIndexTest {
public void testBitSliceIndexMix() {
IntType intType = new IntType();
FieldRef fieldRef = new FieldRef(0, "", intType);
- BitSliceIndexBitmapFileIndex bsiFileIndex = new
BitSliceIndexBitmapFileIndex(intType, null);
+ BitSliceIndexBitmapFileIndex bsiFileIndex = new
BitSliceIndexBitmapFileIndex(intType);
FileIndexWriter writer = bsiFileIndex.createWriter();
Object[] arr = {1, 2, null, -2, -2, -1, null, 2, 0, 5, null};
@@ -111,7 +111,7 @@ public class BitSliceIndexBitmapFileIndexTest {
public void testBitSliceIndexPositiveOnly() {
IntType intType = new IntType();
FieldRef fieldRef = new FieldRef(0, "", intType);
- BitSliceIndexBitmapFileIndex bsiFileIndex = new
BitSliceIndexBitmapFileIndex(intType, null);
+ BitSliceIndexBitmapFileIndex bsiFileIndex = new
BitSliceIndexBitmapFileIndex(intType);
FileIndexWriter writer = bsiFileIndex.createWriter();
Object[] arr = {0, 1, null, 3, 4, 5, 6, 0, null};
@@ -183,7 +183,7 @@ public class BitSliceIndexBitmapFileIndexTest {
public void testBitSliceIndexNegativeOnly() {
IntType intType = new IntType();
FieldRef fieldRef = new FieldRef(0, "", intType);
- BitSliceIndexBitmapFileIndex bsiFileIndex = new
BitSliceIndexBitmapFileIndex(intType, null);
+ BitSliceIndexBitmapFileIndex bsiFileIndex = new
BitSliceIndexBitmapFileIndex(intType);
FileIndexWriter writer = bsiFileIndex.createWriter();
Object[] arr = {null, -1, null, -3, -4, -5, -6, -1, null};
diff --git
a/paimon-common/src/test/java/org/apache/paimon/predicate/BetweenTest.java
b/paimon-common/src/test/java/org/apache/paimon/predicate/BetweenTest.java
new file mode 100644
index 0000000000..f5692d85f5
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/BetweenTest.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class BetweenTest {
+
+ @Test
+ public void testOneLessOrEqualNotRewrite() {
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+ Predicate lte = builder.lessOrEqual(0, 10);
+ Predicate isNotNull = builder.isNotNull(0);
+
+ Predicate result = PredicateBuilder.and(isNotNull, lte);
+
+ assertThat(result).isInstanceOf(CompoundPredicate.class);
+ CompoundPredicate compoundResult = (CompoundPredicate) result;
+ assertThat(compoundResult.function()).isInstanceOf(And.class);
+ assertThat(compoundResult.children()).hasSize(2);
+
+ assertThat(compoundResult.children().get(1)).isEqualTo(lte);
+ assertThat(compoundResult.children().get(0)).isEqualTo(isNotNull);
+ }
+
+ @Test
+ public void testTryRewriteBetweenPredicateBasic() {
+ // Test basic case: AND(a>=1, a<=10, a is not null) should be
rewritten to BETWEEN
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+ Predicate gte = builder.greaterOrEqual(0, 1);
+ Predicate lte = builder.lessOrEqual(0, 10);
+ Predicate isNotNull = builder.isNotNull(0);
+
+ Predicate result = PredicateBuilder.and(gte, isNotNull, lte);
+
+ assertThat(result).isInstanceOf(CompoundPredicate.class);
+ CompoundPredicate compoundResult = (CompoundPredicate) result;
+ assertThat(compoundResult.function()).isInstanceOf(And.class);
+ assertThat(compoundResult.children()).hasSize(2);
+
+ Predicate betweenChild = compoundResult.children().get(1);
+ assertThat(betweenChild).isInstanceOf(LeafPredicate.class);
+ LeafPredicate betweenLeaf = (LeafPredicate) betweenChild;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(1, 10);
+
+ Predicate notNullChild = compoundResult.children().get(0);
+ assertThat(notNullChild).isInstanceOf(LeafPredicate.class);
+ assertThat(notNullChild.toString()).contains("IsNotNull");
+ }
+
+ @Test
+ public void testTryRewriteBetweenPredicateRecursive() {
+ // Test recursive case: OR(b>=1, AND(a>=1, a<=10, a is not null))
should rewrite nested AND
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType(), new IntType()));
+
+ Predicate gteB = builder.greaterOrEqual(1, 1);
+ Predicate gteA = builder.greaterOrEqual(0, 1);
+ Predicate lteA = builder.lessOrEqual(0, 10);
+ Predicate isNotNullA = builder.isNotNull(0);
+ Predicate andPredicate = PredicateBuilder.and(gteA, isNotNullA, lteA);
+ Predicate result = PredicateBuilder.or(gteB, andPredicate);
+
+ assertThat(result).isInstanceOf(CompoundPredicate.class);
+ CompoundPredicate compoundResult = (CompoundPredicate) result;
+ assertThat(compoundResult.function()).isInstanceOf(Or.class);
+ assertThat(compoundResult.children()).hasSize(2);
+
+ Predicate secondChild = compoundResult.children().get(0);
+ assertThat(secondChild).isInstanceOf(LeafPredicate.class);
+ assertThat(secondChild.toString()).contains("GreaterOrEqual");
+
+ Predicate firstChild = compoundResult.children().get(1);
+ assertThat(firstChild).isInstanceOf(CompoundPredicate.class);
+ CompoundPredicate innerAnd = (CompoundPredicate) firstChild;
+ assertThat(innerAnd.function()).isInstanceOf(And.class);
+ assertThat(innerAnd.children()).hasSize(2);
+
+ Predicate betweenCandidate = innerAnd.children().get(1);
+ assertThat(betweenCandidate).isInstanceOf(LeafPredicate.class);
+ LeafPredicate betweenLeaf = (LeafPredicate) betweenCandidate;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(1, 10);
+ }
+
+ /**
+ * Test this complicated scenario.
+ *
+ * <pre>{@code
+ * AND
+ * / | \
+ * OR AND a>=1
+ * /| || \
+ * / | / | \
+ * a>=1 a<=2 OR AND a>=2
+ * / | | \
+ * / | | \
+ * a>=1 b<2 b>=1 a<=10
+ *
+ * }</pre>
+ */
+ @Test
+ public void testAnExtremeComplicatedPredicate() {
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType(), new IntType()));
+ Predicate l3p1 = builder.greaterOrEqual(0, 1);
+ Predicate l3p2 = builder.lessThan(1, 2);
+ Predicate l3p3 = builder.greaterOrEqual(1, 1);
+ Predicate l3p4 = builder.lessOrEqual(0, 10);
+ Predicate l2p1 = builder.greaterOrEqual(0, 1);
+ Predicate l2p2 = builder.lessOrEqual(1, 2);
+ Predicate l2p3 = PredicateBuilder.or(l3p1, l3p2);
+ Predicate l2p4 = PredicateBuilder.and(l3p3, l3p4);
+ Predicate l2p5 = builder.greaterOrEqual(0, 2);
+ Predicate l1p1 = PredicateBuilder.or(l2p1, l2p2);
+ Predicate l1p2 = PredicateBuilder.and(l2p3, l2p4, l2p5);
+ Predicate l1p3 = builder.greaterOrEqual(0, 1);
+ Predicate result = PredicateBuilder.and(l1p1, l1p2, l1p3);
+
+ assertThat(result).isInstanceOf(CompoundPredicate.class);
+
+ CompoundPredicate compoundResult = (CompoundPredicate) result;
+ assertThat(compoundResult.function()).isInstanceOf(And.class);
+
+ // directly check the toString
+ String resultString = compoundResult.toString();
+ assertThat(resultString).contains("Between(f0, [2, 10])");
+ }
+
+ @Test
+ public void testTryRewriteBetweenPredicateIntersection() {
+ // Test intersection case: AND(a>=1, a<=10, a>=2, a<=7) should use
intersection (2, 7)
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate gte1 = builder.greaterOrEqual(0, 1);
+ Predicate lte10 = builder.lessOrEqual(0, 10);
+ Predicate gte2 = builder.greaterOrEqual(0, 2);
+ Predicate lte7 = builder.lessOrEqual(0, 7);
+
+ Predicate result =
+ PredicateBuilder.and(
+ PredicateBuilder.and(gte1, lte10),
PredicateBuilder.and(gte2, lte7));
+ assertThat(result).isInstanceOf(LeafPredicate.class);
+ LeafPredicate betweenLeaf = (LeafPredicate) result;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(2, 7);
+
+ result = PredicateBuilder.and(gte1, lte10, gte2, lte7);
+ assertThat(result).isInstanceOf(LeafPredicate.class);
+ betweenLeaf = (LeafPredicate) result;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(2, 7);
+ }
+
+ @Test
+ public void testTryRewriteBetweenPredicateDifferentColumns() {
+ // Test different columns case: AND(a>=1, b<=10) should not be
rewritten
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType(), new IntType()));
+
+ Predicate gteA = builder.greaterOrEqual(0, 1);
+ Predicate lteB = builder.lessOrEqual(1, 10);
+ Predicate result = PredicateBuilder.and(gteA, lteB);
+
+ assertThat(result).isInstanceOf(CompoundPredicate.class);
+ CompoundPredicate compoundResult = (CompoundPredicate) result;
+ assertThat(compoundResult.function()).isInstanceOf(And.class);
+ assertThat(compoundResult.children()).hasSize(2);
+ assertThat(compoundResult.children().stream().map(Predicate::toString))
+ .containsExactlyInAnyOrderElementsOf(
+ Arrays.asList("GreaterOrEqual(f0, 1)",
"LessOrEqual(f1, 10)"));
+ }
+
+ @Test
+ public void testTryRewriteBetweenPredicateInvalidRange() {
+ // Test invalid range case: AND(a>=10, a<=1) should not be rewritten
to BETWEEN
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate gte = builder.greaterOrEqual(0, 10);
+ Predicate lte = builder.lessOrEqual(0, 1);
+ Predicate result = PredicateBuilder.and(gte, lte);
+
+ assertThat(result).isInstanceOf(CompoundPredicate.class);
+ CompoundPredicate compoundResult = (CompoundPredicate) result;
+ assertThat(compoundResult.function()).isInstanceOf(And.class);
+ assertThat(compoundResult.children()).hasSize(2);
+ assertThat(compoundResult.children().stream().map(Predicate::toString))
+ .containsExactlyInAnyOrderElementsOf(
+ Arrays.asList("GreaterOrEqual(f0, 10)",
"LessOrEqual(f0, 1)"));
+ }
+
+ @Test
+ public void testMergeMultipleBetweensValidIntersection() {
+ // Test merging multiple Between predicates with valid intersection
+ // BETWEEN 1 AND 10 AND BETWEEN 5 AND 15 -> BETWEEN 5 AND 10
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate between1 = builder.between(0, 1, 10);
+ Predicate between2 = builder.between(0, 5, 15);
+ Predicate result = PredicateBuilder.and(between1, between2);
+
+ assertThat(result).isInstanceOf(LeafPredicate.class);
+ LeafPredicate betweenLeaf = (LeafPredicate) result;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(5, 10);
+ }
+
+ @Test
+ public void testMergeMultipleBetweensNoIntersection() {
+ // Test merging multiple Between predicates with no valid intersection
+ // BETWEEN 1 AND 5 AND BETWEEN 10 AND 15 -> keep both
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate between1 = builder.between(0, 1, 5);
+ Predicate between2 = builder.between(0, 10, 15);
+ Predicate result = PredicateBuilder.and(between1, between2);
+
+ assertThat(result).isInstanceOf(CompoundPredicate.class);
+ CompoundPredicate compoundResult = (CompoundPredicate) result;
+ assertThat(compoundResult.function()).isInstanceOf(And.class);
+ assertThat(compoundResult.children()).hasSize(2);
+ }
+
+ @Test
+ public void testMergeMultipleBetweensTouchingIntersection() {
+ // Test merging multiple Between predicates with touching intersection
+ // BETWEEN 1 AND 10 AND BETWEEN 10 AND 20 -> BETWEEN 10 AND 10
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate between1 = builder.between(0, 1, 10);
+ Predicate between2 = builder.between(0, 10, 20);
+ Predicate result = PredicateBuilder.and(between1, between2);
+
+ assertThat(result).isInstanceOf(LeafPredicate.class);
+ LeafPredicate betweenLeaf = (LeafPredicate) result;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(10, 10);
+ }
+
+ @Test
+ public void testMergeThreeBetweens() {
+ // Test merging three Between predicates
+ // BETWEEN 1 AND 20 AND BETWEEN 5 AND 15 AND BETWEEN 8 AND 12 ->
BETWEEN 8 AND 12
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate between1 = builder.between(0, 1, 20);
+ Predicate between2 = builder.between(0, 5, 15);
+ Predicate between3 = builder.between(0, 8, 12);
+ Predicate result = PredicateBuilder.and(between1, between2, between3);
+
+ assertThat(result).isInstanceOf(LeafPredicate.class);
+ LeafPredicate betweenLeaf = (LeafPredicate) result;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(8, 12);
+ }
+
+ @Test
+ public void testSingleBetweenNotModified() {
+ // Test that a single Between predicate is not modified
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate between = builder.between(0, 1, 10);
+ Predicate result = PredicateBuilder.and(between);
+
+ assertThat(result).isInstanceOf(LeafPredicate.class);
+ LeafPredicate betweenLeaf = (LeafPredicate) result;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(1, 10);
+ }
+
+ @Test
+ public void testBetweenWithOtherPredicates() {
+ // Test Between predicate with other predicates
+ // BETWEEN 1 AND 10 AND isNotNull AND Equal
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate between = builder.between(0, 1, 10);
+ Predicate isNotNull = builder.isNotNull(0);
+ Predicate equal = builder.equal(0, 5);
+ Predicate result = PredicateBuilder.and(between, isNotNull, equal);
+
+ List<Predicate> predicates = PredicateBuilder.splitAnd(result);
+ assertThat(predicates).hasSize(3);
+ }
+
+ @Test
+ public void testMergeBetweensFromLessAndGreater() {
+ // Test that Between predicates created from LessOrEqual and
GreaterOrEqual are merged
+ // (a>=1 AND a<=10) AND (a>=2 AND a<=7) -> BETWEEN 2 AND 7
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate gte1 = builder.greaterOrEqual(0, 1);
+ Predicate lte10 = builder.lessOrEqual(0, 10);
+ Predicate gte2 = builder.greaterOrEqual(0, 2);
+ Predicate lte7 = builder.lessOrEqual(0, 7);
+
+ Predicate and1 = PredicateBuilder.and(gte1, lte10);
+ Predicate and2 = PredicateBuilder.and(gte2, lte7);
+ Predicate result = PredicateBuilder.and(and1, and2);
+
+ assertThat(result).isInstanceOf(LeafPredicate.class);
+ LeafPredicate betweenLeaf = (LeafPredicate) result;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(2, 7);
+ }
+
+ @Test
+ public void testMultipleLessOrEqualKeepSmallest() {
+ // Test that when there are multiple LessOrEqual, only the smallest is
kept
+ // a<=10 AND a<=5 AND a>=1 -> BETWEEN 1 AND 5
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate lte10 = builder.lessOrEqual(0, 10);
+ Predicate lte5 = builder.lessOrEqual(0, 5);
+ Predicate gte1 = builder.greaterOrEqual(0, 1);
+
+ Predicate result = PredicateBuilder.and(lte10, lte5, gte1);
+
+ assertThat(result).isInstanceOf(LeafPredicate.class);
+ LeafPredicate betweenLeaf = (LeafPredicate) result;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(1, 5);
+ }
+
+ @Test
+ public void testMultipleGreaterOrEqualKeepLargest() {
+ // Test that when there are multiple GreaterOrEqual, only the largest
is kept
+ // a>=1 AND a>=5 AND a<=10 -> BETWEEN 5 AND 10
+ PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
+
+ Predicate gte1 = builder.greaterOrEqual(0, 1);
+ Predicate gte5 = builder.greaterOrEqual(0, 5);
+ Predicate lte10 = builder.lessOrEqual(0, 10);
+
+ Predicate result = PredicateBuilder.and(gte1, gte5, lte10);
+
+ assertThat(result).isInstanceOf(LeafPredicate.class);
+ LeafPredicate betweenLeaf = (LeafPredicate) result;
+ assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
+ assertThat(betweenLeaf.literals()).containsExactly(5, 10);
+ }
+}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateBuilderTest.java
b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateBuilderTest.java
index b6b72fd259..8318990ab4 100644
---
a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateBuilderTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateBuilderTest.java
@@ -115,10 +115,10 @@ public class PredicateBuilderTest {
.isEqualTo(
Arrays.asList(
child1,
- builder.isNull(3),
- builder.isNull(4),
+ child3,
builder.isNull(5),
- child3));
+ builder.isNull(4),
+ builder.isNull(3)));
}
@Test
diff --git
a/paimon-common/src/test/java/org/apache/paimon/utils/PredicateUtilsTest.java
b/paimon-common/src/test/java/org/apache/paimon/utils/PredicateUtilsTest.java
deleted file mode 100644
index ca299500ba..0000000000
---
a/paimon-common/src/test/java/org/apache/paimon/utils/PredicateUtilsTest.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.utils;
-
-import org.apache.paimon.predicate.And;
-import org.apache.paimon.predicate.Between;
-import org.apache.paimon.predicate.CompoundPredicate;
-import org.apache.paimon.predicate.LeafPredicate;
-import org.apache.paimon.predicate.Or;
-import org.apache.paimon.predicate.Predicate;
-import org.apache.paimon.predicate.PredicateBuilder;
-import org.apache.paimon.types.IntType;
-import org.apache.paimon.types.RowType;
-
-import org.junit.jupiter.api.Test;
-
-import java.util.Arrays;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-/** Test for {@link PredicateUtils}. */
-public class PredicateUtilsTest {
-
- @Test
- public void testTryRewriteBetweenPredicateBasic() {
- // Test basic case: AND(a>=1, a<=10, a is not null) should be
rewritten to BETWEEN
- PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
- Predicate gte = builder.greaterOrEqual(0, 1);
- Predicate lte = builder.lessOrEqual(0, 10);
- Predicate isNotNull = builder.isNotNull(0);
-
- Predicate andPredicate = PredicateBuilder.and(gte, isNotNull, lte);
- Predicate result =
PredicateUtils.tryRewriteBetweenPredicate(andPredicate);
-
- assertThat(result).isInstanceOf(CompoundPredicate.class);
- CompoundPredicate compoundResult = (CompoundPredicate) result;
- assertThat(compoundResult.function()).isInstanceOf(And.class);
- assertThat(compoundResult.children()).hasSize(2);
-
- Predicate betweenChild = compoundResult.children().get(1);
- assertThat(betweenChild).isInstanceOf(LeafPredicate.class);
- LeafPredicate betweenLeaf = (LeafPredicate) betweenChild;
- assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
- assertThat(betweenLeaf.literals()).containsExactly(1, 10);
-
- Predicate notNullChild = compoundResult.children().get(0);
- assertThat(notNullChild).isInstanceOf(LeafPredicate.class);
- assertThat(notNullChild.toString()).contains("IsNotNull");
- }
-
- @Test
- public void testTryRewriteBetweenPredicateRecursive() {
- // Test recursive case: OR(b>=1, AND(a>=1, a<=10, a is not null))
should rewrite nested AND
- PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType(), new IntType()));
-
- Predicate gteB = builder.greaterOrEqual(1, 1);
- Predicate gteA = builder.greaterOrEqual(0, 1);
- Predicate lteA = builder.lessOrEqual(0, 10);
- Predicate isNotNullA = builder.isNotNull(0);
- Predicate andPredicate = PredicateBuilder.and(gteA, isNotNullA, lteA);
- Predicate orPredicate = PredicateBuilder.or(gteB, andPredicate);
-
- Predicate result =
PredicateUtils.tryRewriteBetweenPredicate(orPredicate);
-
- assertThat(result).isInstanceOf(CompoundPredicate.class);
- CompoundPredicate compoundResult = (CompoundPredicate) result;
- assertThat(compoundResult.function()).isInstanceOf(Or.class);
- assertThat(compoundResult.children()).hasSize(2);
-
- Predicate secondChild = compoundResult.children().get(1);
- assertThat(secondChild).isInstanceOf(LeafPredicate.class);
- assertThat(secondChild.toString()).contains("GreaterOrEqual");
-
- Predicate firstChild = compoundResult.children().get(0);
- assertThat(firstChild).isInstanceOf(CompoundPredicate.class);
- CompoundPredicate innerAnd = (CompoundPredicate) firstChild;
- assertThat(innerAnd.function()).isInstanceOf(And.class);
- assertThat(innerAnd.children()).hasSize(2);
-
- Predicate betweenCandidate = innerAnd.children().get(1);
- assertThat(betweenCandidate).isInstanceOf(LeafPredicate.class);
- LeafPredicate betweenLeaf = (LeafPredicate) betweenCandidate;
- assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
- assertThat(betweenLeaf.literals()).containsExactly(1, 10);
- }
-
- /**
- * Test this complicated scenario.
- *
- * <pre>{@code
- * AND
- * / | \
- * OR AND a>=1
- * /| || \
- * / | / | \
- * a>=1 a<=2 OR AND a>=2
- * / | | \
- * / | | \
- * a>=1 b<2 b>=1 a<=10
- *
- * }</pre>
- */
- @Test
- public void testAnExtremeComplicatedPredicate() {
- PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType(), new IntType()));
- Predicate l3p1 = builder.greaterOrEqual(0, 1);
- Predicate l3p2 = builder.lessThan(1, 2);
- Predicate l3p3 = builder.greaterOrEqual(1, 1);
- Predicate l3p4 = builder.lessOrEqual(0, 10);
- Predicate l2p1 = builder.greaterOrEqual(0, 1);
- Predicate l2p2 = builder.lessOrEqual(1, 2);
- Predicate l2p3 = PredicateBuilder.or(l3p1, l3p2);
- Predicate l2p4 = PredicateBuilder.and(l3p3, l3p4);
- Predicate l2p5 = builder.greaterOrEqual(0, 2);
- Predicate l1p1 = PredicateBuilder.or(l2p1, l2p2);
- Predicate l1p2 = PredicateBuilder.and(l2p3, l2p4, l2p5);
- Predicate l1p3 = builder.greaterOrEqual(0, 1);
- Predicate root = PredicateBuilder.and(l1p1, l1p2, l1p3);
-
- Predicate result = PredicateUtils.tryRewriteBetweenPredicate(root);
- assertThat(result).isInstanceOf(CompoundPredicate.class);
-
- CompoundPredicate compoundResult = (CompoundPredicate) result;
- assertThat(compoundResult.function()).isInstanceOf(And.class);
-
- // directly check the toString
- String resultString = compoundResult.toString();
- assertThat(resultString).contains("Between(f0, [2, 10])");
- }
-
- @Test
- public void testTryRewriteBetweenPredicateIntersection() {
- // Test intersection case: AND(a>=1, a<=10, a>=2, a<=7) should use
intersection (2, 7)
- PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
-
- Predicate gte1 = builder.greaterOrEqual(0, 1);
- Predicate lte10 = builder.lessOrEqual(0, 10);
- Predicate gte2 = builder.greaterOrEqual(0, 2);
- Predicate lte7 = builder.lessOrEqual(0, 7);
-
- Predicate predicate =
- PredicateBuilder.and(
- PredicateBuilder.and(gte1, lte10),
PredicateBuilder.and(gte2, lte7));
- Predicate result =
PredicateUtils.tryRewriteBetweenPredicate(predicate);
-
- assertThat(result).isInstanceOf(LeafPredicate.class);
- LeafPredicate betweenLeaf = (LeafPredicate) result;
- assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
- assertThat(betweenLeaf.literals()).containsExactly(2, 7);
- }
-
- @Test
- public void testTryRewriteBetweenPredicateDifferentColumns() {
- // Test different columns case: AND(a>=1, b<=10) should not be
rewritten
- PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType(), new IntType()));
-
- Predicate gteA = builder.greaterOrEqual(0, 1);
- Predicate lteB = builder.lessOrEqual(1, 10);
- Predicate predicate = PredicateBuilder.and(gteA, lteB);
-
- Predicate result =
PredicateUtils.tryRewriteBetweenPredicate(predicate);
-
- assertThat(result).isInstanceOf(CompoundPredicate.class);
- CompoundPredicate compoundResult = (CompoundPredicate) result;
- assertThat(compoundResult.function()).isInstanceOf(And.class);
- assertThat(compoundResult.children()).hasSize(2);
- assertThat(compoundResult.children().stream().map(Predicate::toString))
- .containsExactlyInAnyOrderElementsOf(
- Arrays.asList("GreaterOrEqual(f0, 1)",
"LessOrEqual(f1, 10)"));
- }
-
- @Test
- public void testTryRewriteBetweenPredicateInvalidRange() {
- // Test invalid range case: AND(a>=10, a<=1) should not be rewritten
to BETWEEN
- PredicateBuilder builder = new PredicateBuilder(RowType.of(new
IntType()));
-
- Predicate gte = builder.greaterOrEqual(0, 10);
- Predicate lte = builder.lessOrEqual(0, 1);
- Predicate predicate = PredicateBuilder.and(gte, lte);
-
- Predicate result =
PredicateUtils.tryRewriteBetweenPredicate(predicate);
-
- assertThat(result).isInstanceOf(CompoundPredicate.class);
- CompoundPredicate compoundResult = (CompoundPredicate) result;
- assertThat(compoundResult.function()).isInstanceOf(And.class);
- assertThat(compoundResult.children()).hasSize(2);
- assertThat(compoundResult.children().stream().map(Predicate::toString))
- .containsExactlyInAnyOrderElementsOf(
- Arrays.asList("GreaterOrEqual(f0, 10)",
"LessOrEqual(f0, 1)"));
- }
-}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java
index f8cf39d175..f4f529dc4c 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java
@@ -27,7 +27,6 @@ import org.apache.paimon.predicate.VectorSearch;
import org.apache.paimon.table.InnerTable;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Filter;
-import org.apache.paimon.utils.PredicateUtils;
import org.apache.paimon.utils.Range;
import javax.annotation.Nullable;
@@ -91,7 +90,6 @@ public class ReadBuilderImpl implements ReadBuilder {
} else {
this.filter = PredicateBuilder.and(this.filter, filter);
}
- this.filter = PredicateUtils.tryRewriteBetweenPredicate(this.filter);
return this;
}
