(druid) branch master updated: perf(sql): reduce temporary allocations for large IN filters (#20362)

[frankchen]

This is an automated email from the ASF dual-hosted git repository.

FrankChen021 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git


The following commit(s) were added to refs/heads/master by this push:
     new cfc53e0ad97 perf(sql): reduce temporary allocations for large IN 
filters (#20362)
cfc53e0ad97 is described below

commit cfc53e0ad97d78c8682900f7cf0cd0a9d43c5624
Author: Frank Chen <frank.chen...@outlook.com>
AuthorDate: Sat Oct 17 09:46:05 2026 +0800

    perf(sql): reduce temporary allocations for large IN filters (#20362)
---
 .../druid/benchmark/query/InPlanningBenchmark.java | 31 ++++++++++++++++++++++
 .../builtin/ScalarInArrayOperatorConversion.java   | 15 +++++++++++
 2 files changed, 46 insertions(+)

diff --git 
a/benchmarks/src/test/java/org/apache/druid/benchmark/query/InPlanningBenchmark.java
 
b/benchmarks/src/test/java/org/apache/druid/benchmark/query/InPlanningBenchmark.java
index 44f9ba6c757..eb51af5fda7 100644
--- 
a/benchmarks/src/test/java/org/apache/druid/benchmark/query/InPlanningBenchmark.java
+++ 
b/benchmarks/src/test/java/org/apache/druid/benchmark/query/InPlanningBenchmark.java
@@ -135,6 +135,8 @@ public class InPlanningBenchmark
   private SqlEngine engine;
   @Nullable
   private PlannerFactory plannerFactory;
+  private String planOnlyLongSql;
+  private String planOnlyStringSql;
   private final Closer closer = Closer.create();
 
   @Setup(Level.Trial)
@@ -211,6 +213,9 @@ public class InPlanningBenchmark
         new DruidHookDispatcher()
     );
 
+    planOnlyLongSql = createQuery("select long1 from foo where long1 in ", 
inClauseLiteralsCount, ValueType.LONG);
+    planOnlyStringSql = createQuery("select string1 from foo where string1 in 
", inClauseLiteralsCount, ValueType.STRING);
+
     String prefix = ("explain plan for select long1 from foo where long1 in ");
     final String sql = createQuery(prefix, inClauseLiteralsCount, 
ValueType.LONG);
 
@@ -240,6 +245,22 @@ public class InPlanningBenchmark
     getPlan(sql, blackhole);
   }
 
+  @Benchmark
+  @BenchmarkMode(Mode.AverageTime)
+  @OutputTimeUnit(TimeUnit.MILLISECONDS)
+  public void queryLongInSqlPlanOnly(Blackhole blackhole)
+  {
+    planOnly(planOnlyLongSql, blackhole);
+  }
+
+  @Benchmark
+  @BenchmarkMode(Mode.AverageTime)
+  @OutputTimeUnit(TimeUnit.MILLISECONDS)
+  public void queryStringInSqlPlanOnly(Blackhole blackhole)
+  {
+    planOnly(planOnlyStringSql, blackhole);
+  }
+
   @Benchmark
   @BenchmarkMode(Mode.AverageTime)
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
@@ -345,4 +366,14 @@ public class InPlanningBenchmark
       return resultSequence;
     }
   }
+
+  private void planOnly(final String sql, final Blackhole blackhole)
+  {
+    final Map<String, Object> context = ImmutableMap.of(
+        "inSubQueryThreshold", inSubQueryThreshold, "useCache", false);
+
+    try (final DruidPlanner planner = 
plannerFactory.createPlannerForTesting(engine, sql, context)) {
+      blackhole.consume(planner.plan());
+    }
+  }
 }
diff --git 
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ScalarInArrayOperatorConversion.java
 
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ScalarInArrayOperatorConversion.java
index 33e49697f52..98e2994f92a 100644
--- 
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ScalarInArrayOperatorConversion.java
+++ 
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ScalarInArrayOperatorConversion.java
@@ -21,11 +21,13 @@ package org.apache.druid.sql.calcite.expression.builtin;
 
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.sql.SqlFunction;
 import org.apache.calcite.sql.type.OperandTypes;
 import org.apache.calcite.sql.type.ReturnTypes;
 import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.druid.math.expr.Evals;
 import org.apache.druid.query.extraction.ExtractionFn;
 import org.apache.druid.query.filter.DimFilter;
@@ -111,6 +113,19 @@ public class ScalarInArrayOperatorConversion extends 
DirectOperatorConversion
       final List<Object> arrayElementLiteralValues = new 
ArrayList<>(arrayElements.size());
 
       for (final RexNode arrayElement : arrayElements) {
+        // Extract non-null integer and string RexLiterals directly to avoid 
temporary
+        // DruidLiteral allocations for large IN lists.
+        if (arrayElement instanceof RexLiteral && 
!RexLiteral.isNullLiteral(arrayElement)) {
+          final SqlTypeName sqlTypeName = 
arrayElement.getType().getSqlTypeName();
+          if (SqlTypeName.INT_TYPES.contains(sqlTypeName)) {
+            arrayElementLiteralValues.add(((Number) 
RexLiteral.value(arrayElement)).longValue());
+            continue;
+          } else if (SqlTypeName.STRING_TYPES.contains(sqlTypeName)) {
+            
arrayElementLiteralValues.add(RexLiteral.stringValue(arrayElement));
+            continue;
+          }
+        }
+
         final DruidLiteral arrayElementEval = 
Expressions.calciteLiteralToDruidLiteral(plannerContext, arrayElement);
         if (arrayElementEval == null) {
           return null;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org
For additional commands, e-mail: commits-h...@druid.apache.org