This is an automated email from the ASF dual-hosted git repository.
zhenchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new b9e9e9d958 [CALCITE-7342] Quidem test support for
TopDownGeneralDecorrelator
b9e9e9d958 is described below
commit b9e9e9d958c6f5c4e6bff52796cdb6a4bfff8833
Author: Zhen Chen <[email protected]>
AuthorDate: Tue Dec 30 20:39:16 2025 +0800
[CALCITE-7342] Quidem test support for TopDownGeneralDecorrelator
---
.../calcite/config/CalciteConnectionConfig.java | 3 +
.../config/CalciteConnectionConfigImpl.java | 5 +
.../calcite/config/CalciteConnectionProperty.java | 3 +
.../apache/calcite/prepare/CalcitePrepareImpl.java | 10 ++
.../org/apache/calcite/prepare/PlannerImpl.java | 28 +++---
.../rel/rules/FilterProjectTransposeRule.java | 5 +-
.../apache/calcite/sql2rel/SqlToRelConverter.java | 11 +++
.../java/org/apache/calcite/tools/Programs.java | 25 ++++-
.../org/apache/calcite/test/CoreQuidemTest.java | 56 ++++++-----
.../org/apache/calcite/test/CoreQuidemTest2.java | 70 ++++++++++++++
core/src/test/resources/sql/blank.iq | 16 ++++
core/src/test/resources/sql/conditions.iq | 17 ++++
core/src/test/resources/sql/hep.iq | 13 +++
core/src/test/resources/sql/planner.iq | 11 +++
.../org/apache/calcite/test/MockDdlExecutor.java | 2 +
.../java/org/apache/calcite/test/QuidemTest.java | 104 ++++++++++++---------
16 files changed, 294 insertions(+), 85 deletions(-)
diff --git
a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java
b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java
index 3999a6ce9b..52c8d4cd56 100644
--- a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java
+++ b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java
@@ -110,6 +110,9 @@ public interface CalciteConnectionConfig extends
ConnectionConfig {
boolean lenientOperatorLookup();
/** Returns the value of {@link CalciteConnectionProperty#TOPDOWN_OPT}. */
boolean topDownOpt();
+ /** Returns the value of
+ * {@link CalciteConnectionProperty#TOPDOWN_GENERAL_DECORRELATION_ENABLED}.
*/
+ boolean topDownGeneralDecorrelationEnabled();
/** Returns the value of {@link
CalciteConnectionProperty#META_TABLE_FACTORY},
* or a default meta table factory if not set. If
diff --git
a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java
b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java
index 7d9f9f76d1..221259cef9 100644
---
a/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java
+++
b/core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java
@@ -215,6 +215,11 @@ public boolean isSet(CalciteConnectionProperty property) {
.getBoolean();
}
+ @Override public boolean topDownGeneralDecorrelationEnabled() {
+ return
CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED.wrap(properties)
+ .getBoolean();
+ }
+
@Override public <T> @PolyNull T metaTableFactory(
Class<T> metaTableFactoryClass,
@PolyNull T defaultMetaTableFactory) {
diff --git
a/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java
b/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java
index a9c7627025..32079be2cf 100644
---
a/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java
+++
b/core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java
@@ -154,6 +154,9 @@ public enum CalciteConnectionProperty implements
ConnectionProperty {
* If true (the default), Calcite de-correlates the plan. */
FORCE_DECORRELATE("forceDecorrelate", Type.BOOLEAN, true, false),
+ TOPDOWN_GENERAL_DECORRELATION_ENABLED("topDownGeneralDecorrelationEnabled",
+ Type.BOOLEAN, false, false),
+
/** Type system. The name of a class that implements
* {@link org.apache.calcite.rel.type.RelDataTypeSystem} and has a public
* default constructor or an {@code INSTANCE} constant. */
diff --git
a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
index 6bb21e1ae2..8439fabd5f 100644
--- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
+++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
@@ -107,8 +107,10 @@
import org.apache.calcite.sql2rel.SqlRexConvertletTable;
import org.apache.calcite.sql2rel.SqlToRelConverter;
import org.apache.calcite.sql2rel.StandardConvertletTable;
+import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator;
import org.apache.calcite.tools.FrameworkConfig;
import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.util.ImmutableIntList;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
@@ -1091,6 +1093,9 @@ private PreparedResult prepare_(Supplier<RelNode> fn,
SqlValidator validator,
CatalogReader catalogReader,
SqlToRelConverter.Config config) {
+ config =
+ config.withTopDownGeneralDecorrelationEnabled(
+ context.config().topDownGeneralDecorrelationEnabled());
return new SqlToRelConverter(this, validator, catalogReader, cluster,
convertletTable, config);
}
@@ -1107,6 +1112,11 @@ private PreparedResult prepare_(Supplier<RelNode> fn,
@Override protected RelNode decorrelate(SqlToRelConverter
sqlToRelConverter,
SqlNode query, RelNode rootRel) {
+ if (context.config().topDownGeneralDecorrelationEnabled()) {
+ final RelBuilder relBuilder =
+
sqlToRelConverter.config().getRelBuilderFactory().create(rootRel.getCluster(),
null);
+ return TopDownGeneralDecorrelator.decorrelateQuery(rootRel,
relBuilder);
+ }
return sqlToRelConverter.decorrelate(query, rootRel);
}
diff --git a/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java
b/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java
index ff2903047d..5fece8fa97 100644
--- a/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java
+++ b/core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java
@@ -51,6 +51,7 @@
import org.apache.calcite.sql2rel.RelDecorrelator;
import org.apache.calcite.sql2rel.SqlRexConvertletTable;
import org.apache.calcite.sql2rel.SqlToRelConverter;
+import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator;
import org.apache.calcite.tools.FrameworkConfig;
import org.apache.calcite.tools.Planner;
import org.apache.calcite.tools.Program;
@@ -262,8 +263,10 @@ private void ready() {
final RelOptCluster cluster =
RelOptCluster.create(requireNonNull(planner, "planner"),
rexBuilder);
- final SqlToRelConverter.Config config =
- sqlToRelConverterConfig.withTrimUnusedFields(false);
+ final SqlToRelConverter.Config config = sqlToRelConverterConfig
+ .withTrimUnusedFields(false)
+ .withTopDownGeneralDecorrelationEnabled(
+ connectionConfig.topDownGeneralDecorrelationEnabled());
final SqlToRelConverter sqlToRelConverter =
new SqlToRelConverter(this, validator,
createCatalogReader(), cluster, convertletTable, config);
@@ -272,8 +275,9 @@ private void ready() {
root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true));
final RelBuilder relBuilder =
config.getRelBuilderFactory().create(cluster, null);
- root =
- root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder));
+ root = config.isTopDownGeneralDecorrelationEnabled()
+ ? root.withRel(TopDownGeneralDecorrelator.decorrelateQuery(root.rel,
relBuilder))
+ : root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder));
state = State.STATE_5_CONVERTED;
return root;
}
@@ -314,20 +318,22 @@ public class ViewExpanderImpl implements ViewExpander {
final RexBuilder rexBuilder = createRexBuilder();
final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder);
- final SqlToRelConverter.Config config =
- sqlToRelConverterConfig.withTrimUnusedFields(false);
+ final SqlToRelConverter.Config config = sqlToRelConverterConfig
+ .withTrimUnusedFields(false)
+ .withTopDownGeneralDecorrelationEnabled(
+ connectionConfig.topDownGeneralDecorrelationEnabled());
final SqlToRelConverter sqlToRelConverter =
new SqlToRelConverter(this, validator,
catalogReader, cluster, convertletTable, config);
- final RelRoot root =
+ RelRoot root =
sqlToRelConverter.convertQuery(sqlNode, true, false);
- final RelRoot root2 =
- root.withRel(sqlToRelConverter.flattenTypes(root.rel, true));
+ root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true));
final RelBuilder relBuilder =
config.getRelBuilderFactory().create(cluster, null);
- return root2.withRel(
- RelDecorrelator.decorrelateQuery(root.rel, relBuilder));
+ return config.isTopDownGeneralDecorrelationEnabled()
+ ? root.withRel(TopDownGeneralDecorrelator.decorrelateQuery(root.rel,
relBuilder))
+ : root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder));
}
// CalciteCatalogReader is stateless; no need to store one
diff --git
a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
index 726c07bb14..3e3cf46a11 100644
---
a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
+++
b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
@@ -175,11 +175,10 @@ protected FilterProjectTransposeRule(
final RelNode input = project.getInput();
final RelTraitSet traitSet = filter.getTraitSet()
.replaceIfs(RelCollationTraitDef.INSTANCE,
- () -> Collections.singletonList(
-
input.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE)))
+ () ->
input.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE))
.replaceIfs(RelDistributionTraitDef.INSTANCE,
() -> Collections.singletonList(
-
input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE)));
+
input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE)));
newCondition =
RexUtil.removeNullabilityCast(relBuilder.getTypeFactory(), newCondition);
newFilterRel = filter.copy(traitSet, input, newCondition);
} else {
diff --git
a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index da96b98b06..dd97b2a8fd 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -3966,6 +3966,9 @@ protected boolean enableDecorrelation() {
}
protected RelNode decorrelateQuery(RelNode rootRel) {
+ if (config.isTopDownGeneralDecorrelationEnabled()) {
+ return TopDownGeneralDecorrelator.decorrelateQuery(rootRel, relBuilder);
+ }
return RelDecorrelator.decorrelateQuery(rootRel, relBuilder);
}
@@ -6496,6 +6499,14 @@ public interface Config {
/** Sets {@link #isDecorrelationEnabled()}. */
Config withDecorrelationEnabled(boolean decorrelationEnabled);
+ /** Returns whether to use the top-down general decorrelator. */
+ @Value.Default default boolean isTopDownGeneralDecorrelationEnabled() {
+ return false;
+ }
+
+ /** Sets {@link #isTopDownGeneralDecorrelationEnabled()}. */
+ Config withTopDownGeneralDecorrelationEnabled(boolean
topDownGeneralDecorrelationEnabled);
+
/** Returns the {@code trimUnusedFields} option. Controls whether to trim
* unused fields as part of the conversion process. */
@Value.Default default boolean isTrimUnusedFields() {
diff --git a/core/src/main/java/org/apache/calcite/tools/Programs.java
b/core/src/main/java/org/apache/calcite/tools/Programs.java
index d473e54f4f..4974db3c20 100644
--- a/core/src/main/java/org/apache/calcite/tools/Programs.java
+++ b/core/src/main/java/org/apache/calcite/tools/Programs.java
@@ -46,6 +46,7 @@
import org.apache.calcite.sql2rel.RelDecorrelator;
import org.apache.calcite.sql2rel.RelFieldTrimmer;
import org.apache.calcite.sql2rel.SqlToRelConverter;
+import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator;
import org.apache.calcite.util.Util;
import com.google.common.collect.ImmutableList;
@@ -259,7 +260,26 @@ public static Program subQuery(RelMetadataProvider
metadataProvider) {
CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE,
CoreRules.JOIN_SUB_QUERY_TO_CORRELATE,
CoreRules.PROJECT_OVER_SUM_TO_SUM0_RULE));
- return of(builder.build(), true, metadataProvider);
+ final Program oldProgram = of(builder.build(), true, metadataProvider);
+
+ final HepProgramBuilder newBuilder = HepProgram.builder();
+ newBuilder.addRuleCollection(
+ ImmutableList.of(CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE,
+ CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE,
+ CoreRules.JOIN_SUB_QUERY_TO_CORRELATE,
+ CoreRules.PROJECT_OVER_SUM_TO_SUM0_RULE));
+ final Program newProgram = of(newBuilder.build(), true, metadataProvider);
+
+ return (planner, rel, requiredOutputTraits, materializations, lattices) ->
{
+ final CalciteConnectionConfig config =
+ planner.getContext().maybeUnwrap(CalciteConnectionConfig.class)
+ .orElse(CalciteConnectionConfig.DEFAULT);
+ final Program program = config.topDownGeneralDecorrelationEnabled()
+ ? newProgram
+ : oldProgram;
+ return program.run(planner, rel, requiredOutputTraits, materializations,
+ lattices);
+ };
}
public static Program measure(RelMetadataProvider metadataProvider) {
@@ -430,6 +450,9 @@ private static class DecorrelateProgram implements Program {
if (config.forceDecorrelate()) {
final RelBuilder relBuilder =
RelFactories.LOGICAL_BUILDER.create(rel.getCluster(), null);
+ if (config.topDownGeneralDecorrelationEnabled()) {
+ return TopDownGeneralDecorrelator.decorrelateQuery(rel, relBuilder);
+ }
return RelDecorrelator.decorrelateQuery(rel, relBuilder);
}
return rel;
diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java
b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java
index 1aa6ab1871..b878bfb085 100644
--- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java
+++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java
@@ -41,7 +41,7 @@
/**
* Test that runs every Quidem file in the "core" module as a test.
*/
-class CoreQuidemTest extends QuidemTest {
+public class CoreQuidemTest extends QuidemTest {
/** Runs a test from the command line.
*
* <p>For example:
@@ -57,6 +57,12 @@ public static void main(String[] args) throws Exception {
/** For {@link QuidemTest#test(String)} parameters. */
@Override public Collection<String> getPath() {
+ return data();
+ }
+
+ /** Returns the list of Quidem files to run.
+ * Subclasses can override this method to gradually add files. */
+ protected Collection<String> data() {
// Start with a test file we know exists, then find the directory and list
// its files.
final String first = "sql/agg.iq";
@@ -68,31 +74,31 @@ public static void main(String[] args) throws Exception {
@Override public Connection connect(String name, boolean reference)
throws Exception {
switch (name) {
case "blank":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
- .with(CalciteAssert.SchemaSpec.BLANK)
+ .with(CalciteAssert.SchemaSpec.BLANK))
.connect();
case "scott":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
- .with(CalciteAssert.Config.SCOTT)
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "scott-spark":
discard(CustomTypeSystems.SPARK_TYPE_SYSTEM);
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
.with(CalciteConnectionProperty.TYPE_SYSTEM,
CustomTypeSystems.class.getName() + "#SPARK_TYPE_SYSTEM")
- .with(CalciteAssert.Config.SCOTT)
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "scott-checked-rounding-half-up":
discard(CustomTypeSystems.ROUNDING_MODE_HALF_UP);
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
// Use bigquery conformance, which forces checked arithmetic
@@ -100,84 +106,84 @@ public static void main(String[] args) throws Exception {
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
.with(CalciteConnectionProperty.TYPE_SYSTEM,
CustomTypeSystems.class.getName() + "#ROUNDING_MODE_HALF_UP")
- .with(CalciteAssert.Config.SCOTT)
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "scott-negative-scale":
discard(CustomTypeSystems.NEGATIVE_SCALE);
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
.with(CalciteConnectionProperty.TYPE_SYSTEM,
CustomTypeSystems.class.getName() + "#NEGATIVE_SCALE")
- .with(CalciteAssert.Config.SCOTT)
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "scott-negative-scale-rounding-half-up":
discard(CustomTypeSystems.NEGATIVE_SCALE_ROUNDING_MODE_HALF_UP);
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
.with(CalciteConnectionProperty.TYPE_SYSTEM,
CustomTypeSystems.class.getName()
+ "#NEGATIVE_SCALE_ROUNDING_MODE_HALF_UP")
- .with(CalciteAssert.Config.SCOTT)
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "scott-lenient":
// Same as "scott", but uses LENIENT conformance.
// TODO: add a way to change conformance without defining a new
// connection
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
.with(CalciteConnectionProperty.CONFORMANCE,
SqlConformanceEnum.LENIENT)
- .with(CalciteAssert.Config.SCOTT)
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "scott-babel":
// Same as "scott", but uses BABEL conformance.
// connection
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
.with(CalciteConnectionProperty.CONFORMANCE,
SqlConformanceEnum.BABEL)
- .with(CalciteAssert.Config.SCOTT)
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "scott-mysql":
// Same as "scott", but uses MySQL conformance.
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
.with(CalciteConnectionProperty.CONFORMANCE,
SqlConformanceEnum.MYSQL_5)
- .with(CalciteAssert.Config.SCOTT)
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "scott-oracle":
// Same as "scott", but uses Oracle conformance.
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
.with(CalciteConnectionProperty.CONFORMANCE,
SqlConformanceEnum.ORACLE_10)
- .with(CalciteAssert.Config.SCOTT)
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "scott-mssql":
// Same as "scott", but uses SQL_SERVER_2008 conformance.
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
.with(CalciteConnectionProperty.CONFORMANCE,
SqlConformanceEnum.SQL_SERVER_2008)
- .with(CalciteAssert.Config.SCOTT)
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "steelwheels":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.PARSER_FACTORY,
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
.with(CalciteAssert.SchemaSpec.STEELWHEELS)
- .with(Lex.BIG_QUERY)
+ .with(Lex.BIG_QUERY))
.connect();
default:
return super.connect(name, reference);
diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java
b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java
new file mode 100644
index 0000000000..406bd3bc8f
--- /dev/null
+++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest2.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.test;
+
+import org.apache.calcite.config.CalciteConnectionProperty;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Test that runs Quidem files with the top-down decorrelator enabled.
+ */
+public class CoreQuidemTest2 extends CoreQuidemTest {
+ /** Runs a test from the command line.
+ *
+ * <p>For example:
+ *
+ * <blockquote>
+ * <code>java CoreQuidemTest2 sql/dummy.iq</code>
+ * </blockquote> */
+ public static void main(String[] args) throws Exception {
+ for (String arg : args) {
+ new CoreQuidemTest2().test(arg);
+ }
+ }
+
+ @Override protected Collection<String> data() {
+ final List<String> paths = new ArrayList<>(super.data());
+ // These remove operations are temporary and will be deleted
+ // once the new decorrelator can adapt to all scenarios.
+
+ // TODO: The following files involves UNNEST and LEFT_MARK JOIN
+ paths.remove("sql/agg.iq");
+ paths.remove("sql/measure.iq");
+ paths.remove("sql/unnest.iq");
+ paths.remove("sql/lateral.iq");
+ paths.remove("sql/some.iq");
+ paths.remove("sql/sub-query.iq");
+ paths.remove("sql/scalar.iq");
+ paths.remove("sql/join.iq");
+ paths.remove("sql/spatial.iq");
+ paths.remove("sql/measure-paper.iq");
+ paths.remove("sql/misc.iq");
+ return paths;
+ }
+
+ @Override protected CalciteAssert.AssertThat
customize(CalciteAssert.AssertThat assertThat) {
+ return super.customize(assertThat)
+ .with(CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED,
true);
+ }
+
+ @Override protected boolean useTopDownGeneralDecorrelator() {
+ return true;
+ }
+}
diff --git a/core/src/test/resources/sql/blank.iq
b/core/src/test/resources/sql/blank.iq
index c44c39530a..4053cbb26b 100644
--- a/core/src/test/resources/sql/blank.iq
+++ b/core/src/test/resources/sql/blank.iq
@@ -89,6 +89,7 @@ insert into table2 values (NULL, 1), (2, 1);
# Checked on Oracle
!set lateDecorrelate true
select i, j from table1 where table1.j NOT IN (select i from table2 where
table1.i=table2.j);
+!if (use_old_decorr) {
EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0], expr#9=[=($t3, $t8)],
expr#10=[IS NULL($t1)], expr#11=[IS NOT NULL($t7)], expr#12=[<($t4, $t3)],
expr#13=[OR($t10, $t11, $t12)], expr#14=[IS NOT TRUE($t13)], expr#15=[OR($t9,
$t14)], proj#0..1=[{exprs}], $condition=[$t15])
EnumerableMergeJoin(condition=[AND(=($0, $6), =($1, $5))], joinType=[left])
EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])
@@ -113,7 +114,15 @@ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[0],
expr#9=[=($t3, $t8)], expr#10=[
(0 rows)
!ok
+!}
+# TODO: This error needs to be fixed
+!if (use_new_decorr) {
+Unable to convert LEFT_MARK to Linq4j JoinType
+!error
+!}
+
+!if (use_old_decorr) {
select * from table1 where j not in (select i from table2);
+---+---+
| I | J |
@@ -153,6 +162,13 @@ select * from table1 where j not in (select i from table2)
or j = 3;
(1 row)
!ok
+!}
+
+# TODO: This error needs to be fixed
+!if (use_new_decorr) {
+Unable to convert LEFT_MARK to Linq4j JoinType
+!error
+!}
# [CALCITE-4813] ANY_VALUE assumes that arguments should be comparable
select any_value(r) over(), s from(select array[f, s] r, s from (select 1 as
f, 2 as s) t) t;
diff --git a/core/src/test/resources/sql/conditions.iq
b/core/src/test/resources/sql/conditions.iq
index 5fa94d0889..7ce0c78c9c 100644
--- a/core/src/test/resources/sql/conditions.iq
+++ b/core/src/test/resources/sql/conditions.iq
@@ -418,6 +418,7 @@ where empno = 7369;
!ok
+!if (use_old_decorr) {
EnumerableCalc(expr#0..2=[{inputs}], EMPNO=[$t0])
EnumerableNestedLoopJoin(condition=[true], joinType=[left])
EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL],
expr#9=[7369], expr#10=[=($t8, $t9)], EMPNO=[$t0], $condition=[$t10])
@@ -430,6 +431,22 @@ EnumerableCalc(expr#0..2=[{inputs}], EMPNO=[$t0])
EnumerableAggregate(group=[{}], agg#0=[COUNT()])
EnumerableTableScan(table=[[scott, EMP]])
!plan
+!}
+
+!if (use_new_decorr) {
+EnumerableCalc(expr#0..1=[{inputs}], EMPNO=[$t0])
+ EnumerableNestedLoopJoin(condition=[true], joinType=[left])
+ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t0):INTEGER NOT NULL],
expr#9=[7369], expr#10=[=($t8, $t9)], EMPNO=[$t0], $condition=[$t10])
+ EnumerableTableScan(table=[[scott, EMP]])
+ EnumerableCalc(expr#0..1=[{inputs}], expr#2=[0], DUMMY=[$t2])
+ EnumerableNestedLoopJoin(condition=[true], joinType=[inner])
+ EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0])
+ EnumerableTableScan(table=[[scott, DEPT]])
+ EnumerableCalc(expr#0=[{inputs}], expr#1=[0:BIGINT], expr#2=[>($t0,
$t1)], $f0=[$t0], $condition=[$t2])
+ EnumerableAggregate(group=[{}], agg#0=[COUNT()])
+ EnumerableTableScan(table=[[scott, EMP]])
+!plan
+!}
# sub-query return true with Equal condition
select r.empno, s.deptno
diff --git a/core/src/test/resources/sql/hep.iq
b/core/src/test/resources/sql/hep.iq
index baa57592fa..556d10f721 100644
--- a/core/src/test/resources/sql/hep.iq
+++ b/core/src/test/resources/sql/hep.iq
@@ -141,6 +141,8 @@ WHERE e1.mgr > 12
(5 rows)
!ok
+
+!if (use_old_decorr) {
EnumerableCalc(expr#0..3=[{inputs}], MGR=[$t1], COMM=[$t2])
EnumerableHashJoin(condition=[=($1, $3)], joinType=[inner])
EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER],
expr#9=[12], expr#10=[>($t8, $t9)], EMPNO=[$t0], MGR=[$t3], COMM=[$t6],
$condition=[$t10])
@@ -149,6 +151,17 @@ EnumerableCalc(expr#0..3=[{inputs}], MGR=[$t1], COMM=[$t2])
EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)],
expr#9=[5.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], expr#11=[IS NOT
NULL($t3)], expr#12=[AND($t10, $t11)], MGR=[$t3], $condition=[$t12])
EnumerableTableScan(table=[[scott, EMP]])
!plan
+!}
+
+!if (use_new_decorr) {
+EnumerableCalc(expr#0..2=[{inputs}], MGR=[$t1], COMM=[$t2])
+ EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $4)], joinType=[semi])
+ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t3):INTEGER],
expr#9=[12], expr#10=[>($t8, $t9)], EMPNO=[$t0], MGR=[$t3], COMM=[$t6],
$condition=[$t10])
+ EnumerableTableScan(table=[[scott, EMP]])
+ EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t6):DECIMAL(12, 2)],
expr#9=[5.00:DECIMAL(12, 2)], expr#10=[>($t8, $t9)], expr#11=[IS NOT
NULL($t3)], expr#12=[AND($t10, $t11)], EMPNO=[$t0], MGR=[$t3], COMM=[$t6],
$condition=[$t12])
+ EnumerableTableScan(table=[[scott, EMP]])
+!plan
+!}
!set hep-rules original
# End hep.iq
diff --git a/core/src/test/resources/sql/planner.iq
b/core/src/test/resources/sql/planner.iq
index 0461cc644b..1147a534ef 100644
--- a/core/src/test/resources/sql/planner.iq
+++ b/core/src/test/resources/sql/planner.iq
@@ -359,6 +359,7 @@ or
!ok
+!if (use_old_decorr) {
EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)),
AND(<($5, 255), >=($8, 344))))], joinType=[inner])
EnumerableMergeJoin(condition=[AND(=($0, $3), OR(>($1, 11), <($5, 255)))],
joinType=[inner])
EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 },
{ 4, 14, 144 }, { 5, 15, 155 }]])
@@ -366,6 +367,16 @@ EnumerableHashJoin(condition=[AND(=($0, $6), OR(AND(>($1,
11), <=($7, 32)), AND(
EnumerableCalc(expr#0..2=[{inputs}], expr#3=[32], expr#4=[<=($t1, $t3)],
expr#5=[344], expr#6=[>=($t2, $t5)], expr#7=[OR($t4, $t6)],
proj#0..2=[{exprs}], $condition=[$t7])
EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 },
{ 4, 34, 344 }, { 5, 35, 355 }]])
!plan
+!}
+
+!if (use_new_decorr) {
+EnumerableMergeJoin(condition=[AND(=($0, $6), OR(AND(>($1, 11), <=($7, 32)),
AND(<($5, 255), >=($8, 344))))], joinType=[inner])
+ EnumerableMergeJoin(condition=[=($0, $3)], joinType=[inner])
+ EnumerableValues(tuples=[[{ 1, 11, 111 }, { 2, 12, 122 }, { 3, 13, 133 },
{ 4, 14, 144 }, { 5, 15, 155 }]])
+ EnumerableValues(tuples=[[{ 1, 21, 211 }, { 2, 22, 222 }, { 3, 23, 233 },
{ 4, 24, 244 }, { 5, 25, 255 }]])
+ EnumerableValues(tuples=[[{ 1, 31, 311 }, { 2, 32, 322 }, { 3, 33, 333 }, {
4, 34, 344 }, { 5, 35, 355 }]])
+!plan
+!}
!set planner-rules original
# [CALCITE-7086] Implement a rule that performs the inverse operation of
AggregateCaseToFilterRule
diff --git a/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java
b/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java
index c12618e13a..8d744d829b 100644
--- a/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java
+++ b/testkit/src/main/java/org/apache/calcite/test/MockDdlExecutor.java
@@ -25,6 +25,7 @@
import org.apache.calcite.linq4j.QueryProvider;
import org.apache.calcite.linq4j.Queryable;
import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.plan.Contexts;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -207,6 +208,7 @@ protected static void populate(SqlIdentifier name, SqlNode
query,
requireNonNull(
Schemas.subSchema(context.getRootSchema(),
context.getDefaultSchemaPath())).plus())
+ .context(Contexts.of(context.config()))
.build();
final Planner planner = Frameworks.getPlanner(config);
try {
diff --git a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
index bc0b94f14f..67b43ff462 100644
--- a/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/QuidemTest.java
@@ -145,10 +145,18 @@ public static class ExplainValidatedCommand extends
AbstractCommand {
private static final Pattern PATTERN = Pattern.compile("\\.iq$");
// Saved original planner rules
- private static @Nullable List<RelOptRule> originalRules;
+ private @Nullable List<RelOptRule> originalRules;
- private static @Nullable Object getEnv(String varName) {
+ protected boolean useTopDownGeneralDecorrelator() {
+ return false;
+ }
+
+ private @Nullable Object getEnv(String varName) {
switch (varName) {
+ case "use_old_decorr":
+ return !useTopDownGeneralDecorrelator();
+ case "use_new_decorr":
+ return useTopDownGeneralDecorrelator();
case "jdk18":
return System.getProperty("java.version").startsWith("1.8");
case "fixed":
@@ -200,7 +208,7 @@ protected static Collection<String> data(String first) {
final List<String> paths = new ArrayList<>();
final FilenameFilter filter = new PatternFilenameFilter(".*\\.iq$");
for (File f : Util.first(dir.listFiles(filter), new File[0])) {
- paths.add(f.getAbsolutePath().substring(commonPrefixLength));
+ paths.add(n2u(f.getAbsolutePath().substring(commonPrefixLength)));
}
return paths;
}
@@ -221,7 +229,8 @@ protected void checkRun(String path) throws Exception {
// outFile = "/home/fred/calcite/core/build/quidem/test/sql/agg.iq"
final URL inUrl = QuidemTest.class.getResource("/" + n2u(path));
inFile = Sources.of(requireNonNull(inUrl, "inUrl")).file();
- outFile = replaceDir(inFile, "resources", "quidem");
+ outFile = replaceDir(inFile, "resources", "quidem/"
+ + getClass().getSimpleName());
}
Util.discard(outFile.getParentFile().mkdirs());
try (Reader reader = Util.reader(inFile);
@@ -266,7 +275,7 @@ protected void checkRun(String path) throws Exception {
// - Reset defaults: "original"
if (propertyName.equals("planner-rules")) {
if (value.equals("original")) {
- closer.add(Hook.PLANNER.addThread(QuidemTest::resetPlanner));
+ closer.add(Hook.PLANNER.addThread(this::resetPlanner));
} else {
closer.add(
Hook.PLANNER.addThread((Consumer<RelOptPlanner>)
@@ -314,7 +323,7 @@ protected void checkRun(String path) throws Exception {
}
}
})
- .withEnv(QuidemTest::getEnv)
+ .withEnv(this::getEnv)
.build();
new Quidem(config).execute();
}
@@ -337,7 +346,7 @@ private static void updatePlanner(RelOptPlanner planner,
String value) {
rulesAdd.forEach(planner::addRule);
}
- private static void resetPlanner(RelOptPlanner planner) {
+ private void resetPlanner(RelOptPlanner planner) {
if (originalRules != null) {
planner.getRules().forEach(planner::removeRule);
originalRules.forEach(planner::addRule);
@@ -457,6 +466,11 @@ private static File replaceDir(File file, String target,
String replacement) {
n2u('/' + replacement + '/')));
}
+ /** Allows subclasses to customize the connection. */
+ protected CalciteAssert.AssertThat customize(CalciteAssert.AssertThat
assertThat) {
+ return assertThat;
+ }
+
/** Creates a command handler. */
protected CommandHandler createCommandHandler() {
return Quidem.EMPTY_COMMAND_HANDLER;
@@ -501,7 +515,7 @@ public void test(String path) throws Exception {
protected abstract Collection<String> getPath();
/** Quidem connection factory for Calcite's built-in test schemas. */
- protected static class QuidemConnectionFactory
+ protected class QuidemConnectionFactory
implements Quidem.ConnectionFactory {
public Connection connect(String name) throws Exception {
return connect(name, false);
@@ -523,89 +537,89 @@ public Connection connect(String name) throws Exception {
}
switch (name) {
case "hr":
- return CalciteAssert.hr()
+ return customize(CalciteAssert.hr())
.connect();
case "aux":
- return CalciteAssert.hr()
- .with(CalciteAssert.Config.AUX)
+ return customize(CalciteAssert.hr()
+ .with(CalciteAssert.Config.AUX))
.connect();
case "foodmart":
- return CalciteAssert.that()
- .with(CalciteAssert.Config.FOODMART_CLONE)
+ return customize(CalciteAssert.that()
+ .with(CalciteAssert.Config.FOODMART_CLONE))
.connect();
case "geo":
- return CalciteAssert.that()
- .with(CalciteAssert.Config.GEO)
+ return customize(CalciteAssert.that()
+ .with(CalciteAssert.Config.GEO))
.connect();
case "scott":
- return CalciteAssert.that()
- .with(CalciteAssert.Config.SCOTT)
+ return customize(CalciteAssert.that()
+ .with(CalciteAssert.Config.SCOTT))
.connect();
case "jdbc_scott":
- return CalciteAssert.that()
- .with(CalciteAssert.Config.JDBC_SCOTT)
+ return customize(CalciteAssert.that()
+ .with(CalciteAssert.Config.JDBC_SCOTT))
.connect();
case "steelwheels":
- return CalciteAssert.that()
- .with(CalciteAssert.SchemaSpec.STEELWHEELS)
+ return customize(CalciteAssert.that()
+ .with(CalciteAssert.SchemaSpec.STEELWHEELS))
.connect();
case "jdbc_steelwheels":
- return CalciteAssert.that()
- .with(CalciteAssert.SchemaSpec.JDBC_STEELWHEELS)
+ return customize(CalciteAssert.that()
+ .with(CalciteAssert.SchemaSpec.JDBC_STEELWHEELS))
.connect();
case "post":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteAssert.Config.REGULAR)
- .with(CalciteAssert.SchemaSpec.POST)
+ .with(CalciteAssert.SchemaSpec.POST))
.connect();
case "post-postgresql":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.FUN, "standard,postgresql")
.with(CalciteAssert.Config.REGULAR)
- .with(CalciteAssert.SchemaSpec.POST)
+ .with(CalciteAssert.SchemaSpec.POST))
.connect();
case "post-big-query":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.FUN, "standard,bigquery")
.with(CalciteAssert.Config.REGULAR)
- .with(CalciteAssert.SchemaSpec.POST)
+ .with(CalciteAssert.SchemaSpec.POST))
.connect();
case "mysqlfunc":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.FUN, "mysql")
.with(CalciteAssert.Config.REGULAR)
- .with(CalciteAssert.SchemaSpec.POST)
+ .with(CalciteAssert.SchemaSpec.POST))
.connect();
case "sparkfunc":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.FUN, "spark")
.with(CalciteAssert.Config.REGULAR)
- .with(CalciteAssert.SchemaSpec.POST)
+ .with(CalciteAssert.SchemaSpec.POST))
.connect();
case "oraclefunc":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.FUN, "oracle")
- .with(CalciteAssert.Config.REGULAR)
+ .with(CalciteAssert.Config.REGULAR))
.connect();
case "mssqlfunc":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.FUN, "mssql")
- .with(CalciteAssert.Config.REGULAR)
+ .with(CalciteAssert.Config.REGULAR))
.connect();
case "catchall":
- return CalciteAssert.that()
+ return customize(CalciteAssert.that()
.with(CalciteConnectionProperty.TIME_ZONE, "UTC")
.withSchema("s",
new ReflectiveSchemaWithoutRowCount(
- new CatchallSchema()))
+ new CatchallSchema())))
.connect();
case "orinoco":
- return CalciteAssert.that()
- .with(CalciteAssert.SchemaSpec.ORINOCO)
+ return customize(CalciteAssert.that()
+ .with(CalciteAssert.SchemaSpec.ORINOCO))
.connect();
case "seq":
- final Connection connection = CalciteAssert.that()
- .withSchema("s", new AbstractSchema())
+ final Connection connection = customize(CalciteAssert.that()
+ .withSchema("s", new AbstractSchema()))
.connect();
connection.unwrap(CalciteConnection.class).getRootSchema()
.subSchemas().get("s")
@@ -623,8 +637,8 @@ public Connection connect(String name) throws Exception {
});
return connection;
case "bookstore":
- return CalciteAssert.that()
- .with(CalciteAssert.SchemaSpec.BOOKSTORE)
+ return customize(CalciteAssert.that()
+ .with(CalciteAssert.SchemaSpec.BOOKSTORE))
.connect();
default:
throw new RuntimeException("unknown connection '" + name + "'");
