This is an automated email from the ASF dual-hosted git repository.
9aman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new ac47494e6a3 Allow invertedIndexDistinctCostRatio=0 (#18359)
ac47494e6a3 is described below
commit ac47494e6a3bead0e6bae76e5aa8a44539e6fb26
Author: Xiang Fu <[email protected]>
AuthorDate: Tue Apr 28 20:53:43 2026 -0700
Allow invertedIndexDistinctCostRatio=0 (#18359)
---
.../common/utils/config/QueryOptionsUtils.java | 22 +++++++++++++++++++++-
.../common/utils/config/QueryOptionsUtilsTest.java | 6 ++++--
.../query/InvertedIndexDistinctOperator.java | 11 ++++++++---
.../queries/InvertedIndexDistinctOperatorTest.java | 5 +++++
4 files changed, 38 insertions(+), 6 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
index ba178425d62..11c507639cf 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
@@ -184,10 +184,11 @@ public class QueryOptionsUtils {
/**
* Returns the cost ratio for the inverted-index-based distinct heuristic,
or null if not set.
* The inverted index path is chosen when dictionaryCardinality * costRatio
<= filteredDocCount.
+ * A cost ratio of 0 forces the inverted index path for any non-empty filter
result.
*/
@Nullable
public static Double getInvertedIndexDistinctCostRatio(Map<String, String>
queryOptions) {
- return
checkedParseDoublePositive(QueryOptionKey.INVERTED_INDEX_DISTINCT_COST_RATIO,
+ return
checkedParseDoubleNonNegative(QueryOptionKey.INVERTED_INDEX_DISTINCT_COST_RATIO,
queryOptions.get(QueryOptionKey.INVERTED_INDEX_DISTINCT_COST_RATIO));
}
@@ -621,6 +622,25 @@ public class QueryOptionsUtils {
return value;
}
+ @Nullable
+ private static Double checkedParseDoubleNonNegative(String optionName,
@Nullable String optionValue) {
+ if (optionValue == null) {
+ return null;
+ }
+ double value;
+ try {
+ value = Double.parseDouble(optionValue.trim());
+ } catch (NumberFormatException nfe) {
+ throw new IllegalArgumentException(
+ String.format("%s must be a non-negative number, got: %s",
optionName, optionValue));
+ }
+ if (!Double.isFinite(value) || value < 0) {
+ throw new IllegalArgumentException(
+ String.format("%s must be a non-negative number, got: %s",
optionName, optionValue));
+ }
+ return value;
+ }
+
@Nullable
private static Long checkedParseLongPositive(String optionName, @Nullable
String optionValue) {
return checkedParseLong(optionName, optionValue, 1);
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java
index c25edaf266c..2dc88855e4b 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java
@@ -261,6 +261,8 @@ public class QueryOptionsUtilsTest {
@Test
public void testInvertedIndexDistinctCostRatioValid() {
+ assertEquals(QueryOptionsUtils.getInvertedIndexDistinctCostRatio(
+ Map.of(INVERTED_INDEX_DISTINCT_COST_RATIO, "0")), Double.valueOf(0));
assertEquals(QueryOptionsUtils.getInvertedIndexDistinctCostRatio(
Map.of(INVERTED_INDEX_DISTINCT_COST_RATIO, "2.5")),
Double.valueOf(2.5));
assertEquals(QueryOptionsUtils.getInvertedIndexDistinctCostRatio(
@@ -270,13 +272,13 @@ public class QueryOptionsUtilsTest {
@Test
public void testInvertedIndexDistinctCostRatioRejectsNonFiniteValues() {
- for (String value : new String[]{"NaN", "Infinity", "-Infinity", "0",
"-1"}) {
+ for (String value : new String[]{"NaN", "Infinity", "-Infinity", "-1",
"invalid"}) {
try {
QueryOptionsUtils.getInvertedIndexDistinctCostRatio(Map.of(INVERTED_INDEX_DISTINCT_COST_RATIO,
value));
fail();
} catch (IllegalArgumentException e) {
assertEquals(e.getMessage(),
- INVERTED_INDEX_DISTINCT_COST_RATIO + " must be a positive number,
got: " + value);
+ INVERTED_INDEX_DISTINCT_COST_RATIO + " must be a non-negative
number, got: " + value);
}
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/InvertedIndexDistinctOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/InvertedIndexDistinctOperator.java
index fabf7aa47ba..87abc98a46c 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/query/InvertedIndexDistinctOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/query/InvertedIndexDistinctOperator.java
@@ -85,7 +85,8 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
* </ul>
*
* <p>Enabled via the {@code useIndexBasedDistinctOperator} query option. The
cost ratio can be tuned
- * via the {@code invertedIndexDistinctCostRatio} query option.
+ * via the {@code invertedIndexDistinctCostRatio} query option; setting it to
0 forces the inverted index path
+ * for non-empty filter results.
*/
public class InvertedIndexDistinctOperator extends
BaseOperator<DistinctResultsBlock> {
private static final String EXPLAIN_NAME = "DISTINCT_INVERTED_INDEX";
@@ -173,7 +174,8 @@ public class InvertedIndexDistinctOperator extends
BaseOperator<DistinctResultsB
* <li>dictCard > 10K: costRatio=6 — inverted index wins when
filteredDocs ≥ ~6x dictCard</li>
* </ul>
*
- * <p>Can be overridden at query time via the query option {@code
invertedIndexDistinctCostRatio}.
+ * <p>Can be overridden at query time via the query option {@code
invertedIndexDistinctCostRatio}. Setting it
+ * to 0 forces the inverted index path for non-empty filter results.
*/
static final NavigableMap<Integer, Double> DEFAULT_COST_RATIO_BY_CARDINALITY;
@@ -190,11 +192,14 @@ public class InvertedIndexDistinctOperator extends
BaseOperator<DistinctResultsB
}
private boolean shouldUseBitmapInvertedIndex(int filteredDocCount) {
- int dictionaryCardinality = _dictionary.length();
if (filteredDocCount == 0) {
return false;
}
Double costRatioOverride =
QueryOptionsUtils.getInvertedIndexDistinctCostRatio(_queryContext.getQueryOptions());
+ if (costRatioOverride != null && costRatioOverride == 0.0) {
+ return true;
+ }
+ int dictionaryCardinality = _dictionary.length();
double costRatio = costRatioOverride != null ? costRatioOverride :
getDefaultCostRatio(dictionaryCardinality);
return (double) dictionaryCardinality * costRatio <= filteredDocCount;
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/InvertedIndexDistinctOperatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/InvertedIndexDistinctOperatorTest.java
index 97bf4a879ef..78708dd79b0 100644
---
a/pinot-core/src/test/java/org/apache/pinot/queries/InvertedIndexDistinctOperatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/queries/InvertedIndexDistinctOperatorTest.java
@@ -420,6 +420,11 @@ public class InvertedIndexDistinctOperatorTest extends
BaseQueriesTest {
"SELECT DISTINCT intColumn FROM testTable WHERE intColumn = 0 "
+ OPT + ", invertedIndexDistinctCostRatio=2)")));
+ // costRatio=0: force inverted index for non-empty filters
+ assertTrue(usedInvertedIndex(runDistinct(
+ "SELECT DISTINCT intColumn FROM testTable WHERE intColumn = 0 "
+ + OPT + ", invertedIndexDistinctCostRatio=0)")));
+
// Default costRatio=30: 100*30=3000 <= 10K → inverted
assertTrue(usedInvertedIndex(runDistinct(
"SELECT DISTINCT intColumn FROM testTable WHERE intColumn >= 0 " + OPT
+ ")")));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
