This is an automated email from the ASF dual-hosted git repository.
FrankChen021 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 11065c14a14 perf(sql): reduce SQL planning allocations for large
literal arrays (#20367)
11065c14a14 is described below
commit 11065c14a14499616733ef6c56e1c99cfda3ef5f
Author: Frank Chen <[email protected]>
AuthorDate: Sat Oct 17 21:11:11 2026 +0800
perf(sql): reduce SQL planning allocations for large literal arrays (#20367)
* perf(sql): reduce planning allocation for literal string arrays
* perf(sql): extend literal array reduction to integers
* perf(sql): reuse normalized literal arrays and verify IN semantics
* Update
* perf(sql): restrict literal array fast path to normalized operands
---
.../sql/calcite/planner/DruidRexExecutor.java | 51 +++++
.../apache/druid/sql/calcite/CalciteQueryTest.java | 57 +++++
.../sql/calcite/planner/DruidRexExecutorTest.java | 241 +++++++++++++++++++++
3 files changed, 349 insertions(+)
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
index 378a89f25b8..13991455f56 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
@@ -19,13 +19,16 @@
package org.apache.druid.sql.calcite.planner;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexExecutor;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.druid.error.InvalidSqlInput;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.math.expr.Evals;
@@ -39,6 +42,7 @@ import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.Expressions;
+import javax.annotation.Nullable;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
@@ -67,6 +71,12 @@ public class DruidRexExecutor implements RexExecutor
)
{
for (RexNode constExp : constExps) {
+ final RexNode reducedArray = tryReduceLiteralArray(rexBuilder, constExp);
+ if (reducedArray != null) {
+ reducedValues.add(reducedArray);
+ continue;
+ }
+
final DruidExpression druidExpression = Expressions.toDruidExpression(
plannerContext,
EMPTY_ROW_SIGNATURE,
@@ -246,4 +256,45 @@ public class DruidRexExecutor implements RexExecutor
}
return constExp.toString();
}
+
+ ///
+ /// Reduces literal string and integer arrays without formatting, parsing,
or evaluating a Druid expression.
+ ///
+ /// Reuses the original array call when every literal's type matches the
array component type, ignoring nullability.
+ /// These literals already have the required types and values, so rebuilding
them would only add allocations.
+ /// Nullability differences do not require conversion: a non-null literal
may belong to an array with nullable elements,
+ /// and reusing the call preserves both null values and the original array
type.
+ ///
+ /// Returns null for mismatched element types, unsupported types, or
non-literal operands (including CAST calls).
+ /// These retain the existing Druid expression evaluation path, including
any required type normalization.
+ ///
+ @Nullable
+ @VisibleForTesting
+ static RexNode tryReduceLiteralArray(final RexBuilder rexBuilder, final
RexNode expression)
+ {
+ if (!(expression instanceof RexCall)) {
+ return null;
+ }
+ if (!((RexCall)
expression).getOperator().equals(SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR)) {
+ return null;
+ }
+
+ final SqlTypeName elementType =
expression.getType().getComponentType().getSqlTypeName();
+ final boolean isString = SqlTypeName.CHAR_TYPES.contains(elementType);
+ if (!isString && !SqlTypeName.INT_TYPES.contains(elementType)) {
+ return null;
+ }
+
+ final List<RexNode> operands = ((RexCall) expression).getOperands();
+ for (final RexNode operand : operands) {
+ if (!(operand instanceof RexLiteral)
+ || !SqlTypeUtil.equalSansNullability(
+ rexBuilder.getTypeFactory(),
expression.getType().getComponentType(), operand.getType()
+ )) {
+ return null;
+ }
+ }
+
+ return expression;
+ }
}
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 5858747d379..3320b06296f 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -5907,6 +5907,63 @@ public class CalciteQueryTest extends
BaseCalciteQueryTest
);
}
+ @Test
+ public void testStringInLiteralArraySemantics()
+ {
+ final String values = "('', 'abc', 'abc', 'a''b', '中文', 'abc ',
'nonmatching')";
+ for (final int threshold : new int[]{2, 1000}) {
+ final Map<String, Object> context = QueryContexts.override(
+ QUERY_CONTEXT_DEFAULT,
+ QueryContexts.IN_FUNCTION_THRESHOLD,
+ threshold
+ );
+ // Exercise the array conversion and ordinary IN planning with the same
SQL and expected results.
+ for (final Object[] testCase : new Object[][]{
+ {"dim1 IN " + values, 2L},
+ {"dim1 NOT IN " + values, 4L},
+ {"dim1 IN ('', 'abc', NULL)", 2L},
+ {"dim1 NOT IN ('', 'abc', NULL)", 0L},
+ {"NULLIF(dim1, 'abc') IN ('abc', 'def', 'nonmatching')", 1L},
+ {"NULLIF(dim1, 'abc') NOT IN ('abc', 'def', 'nonmatching')", 4L},
+ {"dim1 IN (CAST('abc' AS VARCHAR(30)), CONCAT('d', 'ef'),
'nonmatching')", 2L}
+ }) {
+ testBuilder()
+ .sql("SELECT COUNT(*) FROM druid.foo WHERE " + testCase[0])
+ .queryContext(context)
+ .expectedResults(ImmutableList.of(new Object[]{testCase[1]}))
+ .run();
+ }
+ }
+ }
+
+ @Test
+ public void testLongInLiteralArraySemantics()
+ {
+ for (final int threshold : new int[]{2, 1000}) {
+ final Map<String, Object> context = QueryContexts.override(
+ QUERY_CONTEXT_DEFAULT,
+ QueryContexts.IN_FUNCTION_THRESHOLD,
+ threshold
+ );
+ for (final Object[] testCase : new Object[][]{
+ {"cnt IN (1, 1, -9223372036854775808, 9223372036854775807)", 6L},
+ {"cnt NOT IN (1, 1, -9223372036854775808, 9223372036854775807)", 0L},
+ {"cnt IN (-9223372036854775808, 0, 9223372036854775807)", 0L},
+ {"cnt IN (1, 2, NULL)", 6L},
+ {"cnt NOT IN (0, 2, NULL)", 0L},
+ {"NULLIF(cnt, 1) IN (1, 2, 3)", 0L},
+ {"NULLIF(cnt, 1) NOT IN (1, 2, 3)", 0L},
+ {"cnt IN (CAST(1 AS BIGINT), CAST(2 AS INTEGER), 3 + 1)", 6L}
+ }) {
+ testBuilder()
+ .sql("SELECT COUNT(*) FROM druid.foo WHERE " + testCase[0])
+ .queryContext(context)
+ .expectedResults(ImmutableList.of(new Object[]{testCase[1]}))
+ .run();
+ }
+ }
+ }
+
@Test
public void
testInFilterWith23Elements_overBothScalarInArrayAndInSubQueryThresholds()
{
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
index 3e271f0d3f7..6ab7bb5e9e1 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
@@ -23,11 +23,14 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.SqlCollation;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
@@ -66,9 +69,13 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
public class DruidRexExecutorTest extends InitializedNullHandlingTest
{
@@ -293,6 +300,240 @@ public class DruidRexExecutorTest extends
InitializedNullHandlingTest
);
}
+ @Test
+ public void testArrayOfStringLiteralsReduction()
+ {
+ for (final SqlTypeName typeName : SqlTypeName.CHAR_TYPES) {
+ assertLiteralArrayReduction(typeName, Arrays.asList("", "a'b",
"back\\slash", "中文", null));
+ assertLiteralArrayReduction(typeName, Collections.singletonList(null));
+ }
+ }
+
+ @Test
+ public void testStringArrayExpressionUsesEvaluator()
+ {
+ final RexNode array = rexBuilder.makeCall(
+ SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR,
+ rexBuilder.makeCall(SqlStdOperatorTable.CONCAT,
rexBuilder.makeLiteral("a"), rexBuilder.makeLiteral("b"))
+ );
+ // CONCAT requires evaluation rather than literal-array reuse.
+ Assertions.assertNull(DruidRexExecutor.tryReduceLiteralArray(rexBuilder,
array));
+ final RexCall reduced = reduceArrayWithEvaluator(array);
+ Assertions.assertEquals(1, reduced.getOperands().size());
+ Assertions.assertEquals("ab",
RexLiteral.stringValue(reduced.getOperands().get(0)));
+ Assertions.assertEquals(SqlTypeName.CHAR,
reduced.getOperands().get(0).getType().getSqlTypeName());
+ Assertions.assertEquals(2,
reduced.getOperands().get(0).getType().getPrecision());
+ }
+
+ @Test
+ public void testIntegerLiteralArraysReduction()
+ {
+ for (final SqlTypeName typeName : SqlTypeName.INT_TYPES) {
+ assertLiteralArrayReduction(
+ typeName,
+ Arrays.asList(BigDecimal.valueOf(-1), BigDecimal.ZERO, null,
BigDecimal.TEN)
+ );
+ }
+ }
+
+ @Test
+ public void testLongLiteralArrayBoundaries()
+ {
+ for (final List<BigDecimal> values : Arrays.<List<BigDecimal>>asList(
+ Arrays.asList(BigDecimal.valueOf(Long.MIN_VALUE), null,
BigDecimal.valueOf(Long.MAX_VALUE)),
+ Collections.singletonList(null)
+ )) {
+ assertLiteralArrayReduction(SqlTypeName.BIGINT, values);
+ }
+ }
+
+ @Test
+ public void testIntegerArrayExpressionUsesEvaluator()
+ {
+ final RexNode array = rexBuilder.makeCall(
+ SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR,
+ rexBuilder.makeCall(SqlStdOperatorTable.PLUS,
rexBuilder.makeBigintLiteral(BigDecimal.ONE),
+ rexBuilder.makeBigintLiteral(BigDecimal.TEN))
+ );
+ // Addition requires evaluation rather than literal-array reuse.
+ Assertions.assertNull(DruidRexExecutor.tryReduceLiteralArray(rexBuilder,
array));
+ final RexCall reduced = reduceArrayWithEvaluator(array);
+ Assertions.assertEquals(1, reduced.getOperands().size());
+ Assertions.assertEquals(BigDecimal.valueOf(11),
RexLiteral.value(reduced.getOperands().get(0)));
+ Assertions.assertEquals(SqlTypeName.BIGINT,
reduced.getOperands().get(0).getType().getSqlTypeName());
+ }
+
+ @Test
+ public void testLiteralArrayReductionRejectsUnsupportedTypes()
+ {
+ for (final SqlTypeName typeName : Arrays.asList(
+ SqlTypeName.DECIMAL,
+ SqlTypeName.DOUBLE,
+ SqlTypeName.BOOLEAN,
+ SqlTypeName.DATE
+ )) {
+ final RelDataType elementType = typeFactory.createSqlType(typeName);
+ final RexNode array = rexBuilder.makeCall(
+ typeFactory.createArrayType(elementType, -1),
SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR,
+ ImmutableList.of(rexBuilder.makeNullLiteral(elementType))
+ );
+ // Unsupported component types retain evaluator fallback, even when
their only element is null.
+ Assertions.assertNull(DruidRexExecutor.tryReduceLiteralArray(rexBuilder,
array), typeName.toString());
+ }
+ // A scalar literal is not an array constructor.
+ Assertions.assertNull(DruidRexExecutor.tryReduceLiteralArray(rexBuilder,
rexBuilder.makeLiteral("a")));
+ // A non-array call is not eligible for this fast path.
+ Assertions.assertNull(DruidRexExecutor.tryReduceLiteralArray(
+ rexBuilder,
+ rexBuilder.makeCall(SqlStdOperatorTable.CONCAT,
rexBuilder.makeLiteral("a"), rexBuilder.makeLiteral("b"))
+ ));
+ }
+
+ @Test
+ public void testLiteralArrayReductionRejectsCastsAndMismatchedOperands()
+ {
+ final RelDataType stringType =
typeFactory.createSqlType(SqlTypeName.VARCHAR, 30);
+ final RelDataType arrayType = typeFactory.createArrayType(stringType, -1);
+ for (final RexNode operand : ImmutableList.of(
+ rexBuilder.makeAbstractCast(stringType, rexBuilder.makeLiteral("a"),
false),
+ rexBuilder.makeBigintLiteral(BigDecimal.ONE)
+ )) {
+ final RexNode array = rexBuilder.makeCall(
+ arrayType,
+ SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR,
+ ImmutableList.of(operand)
+ );
+ // CAST calls and integer operands in a string array cannot be reused as
matching string literals.
+ Assertions.assertNull(DruidRexExecutor.tryReduceLiteralArray(rexBuilder,
array));
+ }
+ }
+
+ @Test
+ public void testLiteralArrayReductionRejectsDifferentElementTypes()
+ {
+ final RelDataType charType = typeFactory.createSqlType(SqlTypeName.CHAR,
3);
+ final RelDataType arrayType = typeFactory.createArrayType(charType, -1);
+ final RexNode array = rexBuilder.makeCall(
+ arrayType,
+ SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR,
+ ImmutableList.of(rexBuilder.makeLiteral("a"),
rexBuilder.makeLiteral("abc"))
+ );
+ // CHAR(1) does not match CHAR(3); the evaluator must perform the required
padding.
+ Assertions.assertNull(DruidRexExecutor.tryReduceLiteralArray(rexBuilder,
array));
+ final RexCall reduced = reduceArrayWithEvaluator(array);
+ Assertions.assertEquals(2, reduced.getOperands().size());
+ Assertions.assertEquals("a ",
RexLiteral.stringValue(reduced.getOperands().get(0)));
+ Assertions.assertEquals("abc",
RexLiteral.stringValue(reduced.getOperands().get(1)));
+ for (final RexNode operand : reduced.getOperands()) {
+ Assertions.assertEquals(SqlTypeName.CHAR,
operand.getType().getSqlTypeName());
+ Assertions.assertEquals(3, operand.getType().getPrecision());
+ }
+ }
+
+ @Test
+ public void testLiteralArrayReductionRejectsMixedIntegerTypes()
+ {
+ final RelDataType arrayType =
typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.BIGINT), -1);
+ final RexNode array = rexBuilder.makeCall(
+ arrayType,
+ SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR,
+ ImmutableList.of(
+ rexBuilder.makeLiteral(BigDecimal.ONE,
typeFactory.createSqlType(SqlTypeName.INTEGER), false),
+ rexBuilder.makeBigintLiteral(BigDecimal.TEN)
+ )
+ );
+ // INTEGER and BIGINT operands require normalization to the BIGINT
component type.
+ Assertions.assertNull(DruidRexExecutor.tryReduceLiteralArray(rexBuilder,
array));
+ final RexCall reduced = reduceArrayWithEvaluator(array);
+ Assertions.assertEquals(2, reduced.getOperands().size());
+ Assertions.assertEquals(BigDecimal.ONE,
RexLiteral.value(reduced.getOperands().get(0)));
+ Assertions.assertEquals(BigDecimal.TEN,
RexLiteral.value(reduced.getOperands().get(1)));
+ for (final RexNode operand : reduced.getOperands()) {
+ Assertions.assertEquals(SqlTypeName.BIGINT,
operand.getType().getSqlTypeName());
+ }
+ }
+
+ @Test
+ public void testLiteralArrayReductionRejectsDifferentCharsets()
+ {
+ assertCharacterTypeMismatch(StandardCharsets.ISO_8859_1, "primary");
+ }
+
+ @Test
+ public void testLiteralArrayReductionRejectsDifferentCollations()
+ {
+ assertCharacterTypeMismatch(StandardCharsets.UTF_8, "tertiary");
+ }
+
+ private void assertCharacterTypeMismatch(final Charset operandCharset, final
String operandStrength)
+ {
+ final RelDataType charType = typeFactory.createSqlType(SqlTypeName.CHAR,
1);
+ final RelDataType componentType =
typeFactory.createTypeWithCharsetAndCollation(
+ charType,
+ StandardCharsets.UTF_8,
+ new SqlCollation(SqlCollation.Coercibility.IMPLICIT, Locale.US,
StandardCharsets.UTF_8, "primary")
+ );
+ final RelDataType operandType =
typeFactory.createTypeWithCharsetAndCollation(
+ charType,
+ operandCharset,
+ new SqlCollation(SqlCollation.Coercibility.IMPLICIT, Locale.US,
operandCharset, operandStrength)
+ );
+ final RexNode operand = rexBuilder.makeLiteral("a", operandType, false);
+ Assertions.assertInstanceOf(RexLiteral.class, operand);
+ Assertions.assertEquals(operandType, operand.getType());
+ Assertions.assertNotEquals(componentType, operand.getType());
+ final RelDataType arrayType = typeFactory.createArrayType(componentType,
-1);
+ final RexNode array = rexBuilder.makeCall(
+ arrayType,
+ SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR,
+ ImmutableList.of(operand)
+ );
+ // Matching CHAR lengths are insufficient when the charset or collation
differs.
+ Assertions.assertNull(DruidRexExecutor.tryReduceLiteralArray(rexBuilder,
array));
+ final RexCall reduced = reduceArrayWithEvaluator(array);
+ Assertions.assertEquals(1, reduced.getOperands().size());
+ final RexNode reducedOperand = reduced.getOperands().get(0);
+ Assertions.assertEquals("a", RexLiteral.stringValue(reducedOperand));
+ Assertions.assertEquals(componentType, reducedOperand.getType());
+ }
+
+ private void assertLiteralArrayReduction(final SqlTypeName typeName, final
List<?> values)
+ {
+ final RelDataType elementType = SqlTypeName.CHAR_TYPES.contains(typeName)
+ ? typeFactory.createSqlType(typeName, 30)
+ : typeFactory.createSqlType(typeName);
+ final RelDataType arrayType = typeFactory.createArrayType(
+ typeFactory.createTypeWithNullability(elementType, true), -1
+ );
+ final List<RexNode> operands = new ArrayList<>();
+ for (final Object value : values) {
+ final RexNode operand = value == null
+ ? rexBuilder.makeNullLiteral(elementType)
+ : rexBuilder.makeLiteral(value, elementType,
false);
+ Assertions.assertInstanceOf(RexLiteral.class, operand);
+ operands.add(operand);
+ }
+ final RexNode array = rexBuilder.makeCall(arrayType,
SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, operands);
+ final RexNode reduced = DruidRexExecutor.tryReduceLiteralArray(rexBuilder,
array);
+ if (typeName == SqlTypeName.VARCHAR && values.stream().anyMatch(value ->
value != null)) {
+ // makeLiteral(..., false) creates CHAR literals here, which do not
match the VARCHAR component type.
+ Assertions.assertNull(reduced);
+ } else {
+ // Matching literals, including typed nulls, reuse the exact array
despite nullability differences.
+ Assertions.assertSame(array, reduced);
+ }
+ }
+
+ private RexCall reduceArrayWithEvaluator(final RexNode array)
+ {
+ final List<RexNode> reduced = new ArrayList<>();
+ new DruidRexExecutor(PLANNER_CONTEXT).reduce(rexBuilder,
ImmutableList.of(array), reduced);
+ Assertions.assertEquals(1, reduced.size());
+ final RexCall reducedArray = Assertions.assertInstanceOf(RexCall.class,
reduced.get(0));
+ Assertions.assertEquals(SqlKind.ARRAY_VALUE_CONSTRUCTOR,
reducedArray.getKind());
+ return reducedArray;
+ }
+
@Test
public void testMultiValueStringNotReduced()
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
