This is an automated email from the ASF dual-hosted git repository.
gianm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 62b98500a42 fix: Erroneous pruning for non-STRING typed filters.
(#19415)
62b98500a42 is described below
commit 62b98500a427d5161643ef84dc194ba2433803c4
Author: Gian Merlino <[email protected]>
AuthorDate: Wed May 6 09:49:32 2026 -0700
fix: Erroneous pruning for non-STRING typed filters. (#19415)
Fixes #19408 by skipping pruning for RangeFilter with matchValueType
other than STRING.
---
.../apache/druid/query/filter/EqualityFilter.java | 5 +++
.../org/apache/druid/query/filter/RangeFilter.java | 22 ++++++-----
.../apache/druid/query/filter/TypedInFilter.java | 5 +++
.../query/filter/FilterSegmentPrunerTest.java | 43 ++++++++++++++++++++++
.../druid/segment/filter/EqualityFilterTests.java | 24 +++++++++---
.../apache/druid/segment/filter/InFilterTests.java | 27 +++++---------
.../druid/segment/filter/RangeFilterTests.java | 18 +++++----
7 files changed, 103 insertions(+), 41 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/EqualityFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/EqualityFilter.java
index c7a70f03c05..482a96c9906 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/EqualityFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/EqualityFilter.java
@@ -214,6 +214,11 @@ public class EqualityFilter extends
AbstractOptimizableDimFilter implements Filt
if (!Objects.equals(getColumn(), dimension)) {
return null;
}
+ // The RangeSet returned here is compared lexicographically against shard
boundaries (see DimensionRangeShardSpec),
+ // so it is only OK to return one for STRING comparison.
+ if (!matchValueType.is(ValueType.STRING)) {
+ return null;
+ }
RangeSet<String> retSet = TreeRangeSet.create();
if (matchValueEval.isArray()) {
retSet.add(Range.singleton(Arrays.deepToString(matchValueEval.asArray())));
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/RangeFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/RangeFilter.java
index 527b5912208..0d3aeb344ae 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/RangeFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/RangeFilter.java
@@ -271,23 +271,25 @@ public class RangeFilter extends
AbstractOptimizableDimFilter implements Filter
return null;
}
- // We need to return a RangeSet<String>, but we have Object, not String.
We align with the interface by
- // converting things to String, but we'd probably be better off adjusting
the interface to something that is
- // more type aware in the future
+ // The RangeSet returned here is compared lexicographically against shard
boundaries (see DimensionRangeShardSpec),
+ // so it is only OK to return one for STRING comparison.
+ if (!matchValueType.is(ValueType.STRING)) {
+ return null;
+ }
- final Supplier<String> lowerString = () -> lowerEval.isArray() ?
Arrays.deepToString(lowerEval.asArray()) : lowerEval.asString();
- final Supplier<String> upperString = () -> upperEval.isArray() ?
Arrays.deepToString(upperEval.asArray()) : upperEval.asString();
- RangeSet<String> retSet = TreeRangeSet.create();
+ final String lowerString = hasLowerBound() ? lowerEval.asString() : null;
+ final String upperString = hasUpperBound() ? upperEval.asString() : null;
+ final RangeSet<String> retSet = TreeRangeSet.create();
final Range<String> range;
if (!hasLowerBound()) {
- range = isUpperOpen() ? Range.lessThan(upperString.get()) :
Range.atMost(upperString.get());
+ range = isUpperOpen() ? Range.lessThan(upperString) :
Range.atMost(upperString);
} else if (!hasUpperBound()) {
- range = isLowerOpen() ? Range.greaterThan(lowerString.get()) :
Range.atLeast(lowerString.get());
+ range = isLowerOpen() ? Range.greaterThan(lowerString) :
Range.atLeast(lowerString);
} else {
range = Range.range(
- lowerString.get(),
+ lowerString,
isLowerOpen() ? BoundType.OPEN : BoundType.CLOSED,
- upperString.get(),
+ upperString,
isUpperOpen() ? BoundType.OPEN : BoundType.CLOSED
);
}
diff --git
a/processing/src/main/java/org/apache/druid/query/filter/TypedInFilter.java
b/processing/src/main/java/org/apache/druid/query/filter/TypedInFilter.java
index 247c0dcaada..8bcd3e7075f 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/TypedInFilter.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/TypedInFilter.java
@@ -258,6 +258,11 @@ public class TypedInFilter extends
AbstractOptimizableDimFilter implements Filte
if (!Objects.equals(getColumn(), dimension)) {
return null;
}
+ // The RangeSet returned here is compared lexicographically against shard
boundaries (see DimensionRangeShardSpec),
+ // so it is only OK to return one for STRING comparison.
+ if (!matchValueType.is(ValueType.STRING)) {
+ return null;
+ }
RangeSet<String> retSet = TreeRangeSet.create();
for (Object value : sortedMatchValues.get()) {
String valueEquivalent = Evals.asString(value);
diff --git
a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java
b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java
index 63da8751544..d5c0d38beba 100644
---
a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java
+++
b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java
@@ -286,6 +286,49 @@ class FilterSegmentPrunerTest
.verify();
}
+ @Test
+ void testPruneNumericRange()
+ {
+ // Regression test for https://github.com/apache/druid/issues/19408
+ final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z";
+ final DataSegment segLargeIds = makeDataSegment(interval, makeRange("id",
0, "100", "200"));
+
+ final DimFilter filter = new RangeFilter("id", ColumnType.LONG, 80L, null,
false, false, null);
+ final FilterSegmentPruner pruner = new FilterSegmentPruner(filter, null,
null);
+
+ Assertions.assertTrue(pruner.include(segLargeIds));
+ }
+
+ @Test
+ void testPruneNumericEquality()
+ {
+ // Regression test for https://github.com/apache/druid/issues/19408. The
string "1" is not within the
+ // ["1.1", "1.9"] shard range, but the LONG value 1L matches rows with
values like "1.1" or "1.9", so the
+ // segment must not be pruned.
+ final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z";
+ final DataSegment seg = makeDataSegment(interval, makeRange("id", 0,
"1.1", "1.9"));
+
+ final DimFilter filter = new EqualityFilter("id", ColumnType.LONG, 1L,
null);
+ final FilterSegmentPruner pruner = new FilterSegmentPruner(filter, null,
null);
+
+ Assertions.assertTrue(pruner.include(seg));
+ }
+
+ @Test
+ void testPruneNumericIn()
+ {
+ // Regression test for https://github.com/apache/druid/issues/19408. The
strings "1" and "2" are not within the
+ // ["1.1", "1.9"] shard range, but the LONG values 1L and 2L match rows
with values like "1.1" or "1.9", so the
+ // segment must not be pruned.
+ final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z";
+ final DataSegment seg = makeDataSegment(interval, makeRange("id", 0,
"1.1", "1.9"));
+
+ final DimFilter filter = new TypedInFilter("id", ColumnType.LONG,
List.of(1L, 2L), null, null);
+ final FilterSegmentPruner pruner = new FilterSegmentPruner(filter, null,
null);
+
+ Assertions.assertTrue(pruner.include(seg));
+ }
+
private ShardSpec makeRange(
String column,
int partitionNumber,
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
index aa35112b67c..375e16e7fc5 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java
@@ -1602,12 +1602,24 @@ public class EqualityFilterTests
Assert.assertEquals(set, filter.getDimensionRangeSet("x"));
Assert.assertNull(filter.getDimensionRangeSet("y"));
- ExprEval<?> eval = ExprEval.ofType(ExpressionType.STRING_ARRAY, new
Object[]{"abc", "def"});
- filter = new EqualityFilter("x", ColumnType.STRING_ARRAY, eval.value(),
null);
- set = TreeRangeSet.create();
- set.add(Range.singleton(Arrays.deepToString(eval.asArray())));
- Assert.assertEquals(set, filter.getDimensionRangeSet("x"));
- Assert.assertNull(filter.getDimensionRangeSet("y"));
+ // Non-STRING match value types must not return a RangeSet.
+ Assert.assertNull(
+ new EqualityFilter("x", ColumnType.LONG, 1L,
null).getDimensionRangeSet("x")
+ );
+ Assert.assertNull(
+ new EqualityFilter("x", ColumnType.DOUBLE, 1.5,
null).getDimensionRangeSet("x")
+ );
+ Assert.assertNull(
+ new EqualityFilter("x", ColumnType.FLOAT, 1.5f,
null).getDimensionRangeSet("x")
+ );
+ Assert.assertNull(
+ new EqualityFilter(
+ "x",
+ ColumnType.STRING_ARRAY,
+ ExprEval.ofType(ExpressionType.STRING_ARRAY, new Object[]{"abc",
"def"}).value(),
+ null
+ ).getDimensionRangeSet("x")
+ );
}
@Test
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/InFilterTests.java
b/processing/src/test/java/org/apache/druid/segment/filter/InFilterTests.java
index 67dcdde27af..31d87502363 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/InFilterTests.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/InFilterTests.java
@@ -686,23 +686,16 @@ public class InFilterTests
RangeSet<String> range = filter.getDimensionRangeSet("x");
Assert.assertTrue(range.contains("b"));
- filter = inFilter("x", ColumnType.LONG, Arrays.asList(null, 1L, 2L, 3L));
- filter2 = inFilter("x", ColumnType.LONG, Arrays.asList(3L, 1L, null,
2L));
- Assert.assertEquals(filter.getDimensionRangeSet("x"),
filter2.getDimensionRangeSet("x"));
- range = filter.getDimensionRangeSet("x");
- Assert.assertTrue(range.contains("2"));
-
- filter = inFilter("x", ColumnType.DOUBLE, Arrays.asList(null, 1.1, 2.2,
3.3));
- filter2 = inFilter("x", ColumnType.DOUBLE, Arrays.asList(3.3, 1.1, null,
2.2));
- range = filter.getDimensionRangeSet("x");
- Assert.assertEquals(filter.getDimensionRangeSet("x"),
filter2.getDimensionRangeSet("x"));
- Assert.assertTrue(range.contains("2.2"));
-
- filter = inFilter("x", ColumnType.FLOAT, Arrays.asList(null, 1.1f, 2.2f,
3.3f));
- filter2 = inFilter("x", ColumnType.FLOAT, Arrays.asList(3.3f, 1.1f,
null, 2.2f));
- range = filter.getDimensionRangeSet("x");
- Assert.assertEquals(filter.getDimensionRangeSet("x"),
filter2.getDimensionRangeSet("x"));
- Assert.assertTrue(range.contains("2.2"));
+ // Non-STRING match value types must not return a RangeSet.
+ Assert.assertNull(
+ inFilter("x", ColumnType.LONG, Arrays.asList(null, 1L, 2L,
3L)).getDimensionRangeSet("x")
+ );
+ Assert.assertNull(
+ inFilter("x", ColumnType.DOUBLE, Arrays.asList(null, 1.1, 2.2,
3.3)).getDimensionRangeSet("x")
+ );
+ Assert.assertNull(
+ inFilter("x", ColumnType.FLOAT, Arrays.asList(null, 1.1f, 2.2f,
3.3f)).getDimensionRangeSet("x")
+ );
}
@Test
diff --git
a/processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTests.java
b/processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTests.java
index da3f8b31aa9..efef85c952c 100644
---
a/processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTests.java
+++
b/processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTests.java
@@ -35,8 +35,6 @@ import org.apache.druid.error.DruidException;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.math.expr.ExprEval;
-import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.FilterTuning;
import org.apache.druid.query.filter.NotDimFilter;
@@ -2035,12 +2033,16 @@ public class RangeFilterTests
Assert.assertEquals(set, filter.getDimensionRangeSet("x"));
Assert.assertNull(filter.getDimensionRangeSet("y"));
- ExprEval<?> evalLower = ExprEval.ofType(ExpressionType.STRING_ARRAY, new
Object[]{"abc", "def"});
- filter = new RangeFilter("x", ColumnType.STRING_ARRAY,
evalLower.value(), null, true, false, null);
- set = TreeRangeSet.create();
- set.add(Range.greaterThan(Arrays.deepToString(evalLower.asArray())));
- Assert.assertEquals(set, filter.getDimensionRangeSet("x"));
- Assert.assertNull(filter.getDimensionRangeSet("y"));
+ // Non-STRING match value types must not return a RangeSet.
+ Assert.assertNull(
+ new RangeFilter("x", ColumnType.LONG, 80L, null, false, false,
null).getDimensionRangeSet("x")
+ );
+ Assert.assertNull(
+ new RangeFilter("x", ColumnType.DOUBLE, 1.5, 2.5, false, false,
null).getDimensionRangeSet("x")
+ );
+ Assert.assertNull(
+ new RangeFilter("x", ColumnType.FLOAT, null, 7.5f, false, true,
null).getDimensionRangeSet("x")
+ );
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
