[CALCITE-1682] New metadata providers for expression column origin and all predicates in plan
Includes: * RelNode type metadata provider * Ranges containment-based simplification in conjunctive predicates Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/41b05d78 Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/41b05d78 Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/41b05d78 Branch: refs/heads/master Commit: 41b05d784fd2e0ae81b09d013ef8a746036ca446 Parents: 478de56 Author: Jesus Camacho Rodriguez <[email protected]> Authored: Fri Mar 10 12:53:51 2017 +0000 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Wed Apr 26 19:56:37 2017 +0100 ---------------------------------------------------------------------- .../org/apache/calcite/plan/RelOptCluster.java | 1 + .../org/apache/calcite/plan/RelOptUtil.java | 26 +- .../calcite/rel/metadata/BuiltInMetadata.java | 57 +- .../metadata/DefaultRelMetadataProvider.java | 3 + .../rel/metadata/RelMdAllPredicates.java | 249 +++++++ .../rel/metadata/RelMdExpressionLineage.java | 450 +++++++++++++ .../calcite/rel/metadata/RelMdNodeTypes.java | 154 +++++ .../apache/calcite/rel/metadata/RelMdUtil.java | 1 + .../calcite/rel/metadata/RelMetadataQuery.java | 60 ++ .../calcite/rel/metadata/RelTableRef.java | 62 ++ .../org/apache/calcite/rex/LogicVisitor.java | 4 + .../org/apache/calcite/rex/RexBiVisitor.java | 2 + .../java/org/apache/calcite/rex/RexShuttle.java | 4 + .../org/apache/calcite/rex/RexSimplify.java | 274 +++++++- .../apache/calcite/rex/RexTableInputRef.java | 82 +++ .../java/org/apache/calcite/rex/RexUtil.java | 82 +++ .../java/org/apache/calcite/rex/RexVisitor.java | 2 + .../org/apache/calcite/rex/RexVisitorImpl.java | 4 + .../java/org/apache/calcite/sql/SqlKind.java | 7 + .../org/apache/calcite/util/BuiltInMethod.java | 6 + .../apache/calcite/test/RelMetadataTest.java | 647 +++++++++++++++++++ .../org/apache/calcite/test/RelOptRulesTest.xml | 2 +- 22 files changed, 2151 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java b/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java index f88e232..034d978 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptCluster.java @@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicInteger; * optimization of a query. */ public class RelOptCluster { + //~ Instance fields -------------------------------------------------------- private final RelDataTypeFactory typeFactory; http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 9af225e..662d316 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -118,10 +118,12 @@ import java.io.StringWriter; import java.util.AbstractList; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -224,20 +226,32 @@ public abstract class RelOptUtil { * Returns a list of all tables used by this expression or its children */ public static List<RelOptTable> findAllTables(RelNode rel) { + final Multimap<Class<? extends RelNode>, RelNode> nodes = + RelMetadataQuery.instance().getNodeTypes(rel); final List<RelOptTable> usedTables = new ArrayList<>(); - new RelVisitor() { - @Override public void visit(RelNode node, int ordinal, RelNode parent) { - if (node instanceof TableScan) { + for (Entry<Class<? extends RelNode>, Collection<RelNode>> e : nodes.asMap().entrySet()) { + if (TableScan.class.isAssignableFrom(e.getKey())) { + for (RelNode node : e.getValue()) { usedTables.add(node.getTable()); } - super.visit(node, ordinal, parent); } - // CHECKSTYLE: IGNORE 1 - }.go(rel); + } return usedTables; } /** + * Returns a list of all tables used by this expression or its children + */ + public static List<String> findAllTableQualifiedNames(RelNode rel) { + return Lists.transform(findAllTables(rel), + new Function<RelOptTable, String>() { + @Override public String apply(RelOptTable arg0) { + return arg0.getQualifiedName().toString(); + } + }); + } + + /** * Returns a list of variables set by a relational expression or its * descendants. */ http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java index eb1ff79..0e0cbca 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java @@ -27,6 +27,7 @@ import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.ImmutableBitSet; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multimap; import java.util.List; import java.util.Set; @@ -160,6 +161,22 @@ public abstract class BuiltInMetadata { } } + /** Metadata about the node types and count in a relational expression. */ + public interface NodeTypes extends Metadata { + MetadataDef<NodeTypes> DEF = MetadataDef.of(NodeTypes.class, + NodeTypes.Handler.class, BuiltInMethod.NODE_TYPES.method); + + /** + * + */ + Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(); + + /** Handler API. */ + interface Handler extends MetadataHandler<NodeTypes> { + Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(RelNode r, RelMetadataQuery mq); + } + } + /** Metadata about the number of rows returned by a relational expression. */ public interface RowCount extends Metadata { MetadataDef<RowCount> DEF = MetadataDef.of(RowCount.class, @@ -370,6 +387,23 @@ public abstract class BuiltInMetadata { } } + /** Metadata about the origins of expressions. */ + public interface ExpressionLineage extends Metadata { + MetadataDef<ExpressionLineage> DEF = MetadataDef.of(ExpressionLineage.class, + ExpressionLineage.Handler.class, BuiltInMethod.EXPRESSION_LINEAGE.method); + + /** + * + */ + Set<RexNode> getExpressionLineage(RexNode expression); + + /** Handler API. */ + interface Handler extends MetadataHandler<ExpressionLineage> { + Set<RexNode> getExpressionLineage(RelNode r, RelMetadataQuery mq, + RexNode expression); + } + } + /** Metadata about the cost of evaluating a relational expression, including * all of its inputs. */ public interface CumulativeCost extends Metadata { @@ -461,6 +495,26 @@ public abstract class BuiltInMetadata { } } + /** Metadata about the predicates that hold in the rows emitted from a + * relational expression. */ + public interface AllPredicates extends Metadata { + MetadataDef<AllPredicates> DEF = MetadataDef.of(AllPredicates.class, + AllPredicates.Handler.class, BuiltInMethod.ALL_PREDICATES.method); + + /** + * Derives the predicates that hold on rows emitted from a relational + * expression. + * + * @return Predicate list + */ + RelOptPredicateList getAllPredicates(); + + /** Handler API. */ + interface Handler extends MetadataHandler<AllPredicates> { + RelOptPredicateList getAllPredicates(RelNode r, RelMetadataQuery mq); + } + } + /** Metadata about the degree of parallelism of a relational expression, and * how its operators are assigned to processes with independent resource * pools. */ @@ -547,7 +601,8 @@ public abstract class BuiltInMetadata { /** The built-in forms of metadata. */ interface All extends Selectivity, UniqueKeys, RowCount, DistinctRowCount, PercentageOriginalRows, ColumnUniqueness, ColumnOrigin, Predicates, - Collation, Distribution, Size, Parallelism, Memory { + Collation, Distribution, Size, Parallelism, Memory, AllPredicates, + ExpressionLineage, NodeTypes { } } http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rel/metadata/DefaultRelMetadataProvider.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/DefaultRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/DefaultRelMetadataProvider.java index bc04c4e..cb86698 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/DefaultRelMetadataProvider.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/DefaultRelMetadataProvider.java @@ -43,6 +43,8 @@ public class DefaultRelMetadataProvider extends ChainedRelMetadataProvider { ImmutableList.of( RelMdPercentageOriginalRows.SOURCE, RelMdColumnOrigins.SOURCE, + RelMdExpressionLineage.SOURCE, + RelMdNodeTypes.SOURCE, RelMdRowCount.SOURCE, RelMdMaxRowCount.SOURCE, RelMdMinRowCount.SOURCE, @@ -57,6 +59,7 @@ public class DefaultRelMetadataProvider extends ChainedRelMetadataProvider { RelMdSelectivity.SOURCE, RelMdExplainVisibility.SOURCE, RelMdPredicates.SOURCE, + RelMdAllPredicates.SOURCE, RelMdCollation.SOURCE)); } } http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rel/metadata/RelMdAllPredicates.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdAllPredicates.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdAllPredicates.java new file mode 100644 index 0000000..5f87a05 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdAllPredicates.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.metadata; + +import org.apache.calcite.plan.RelOptPredicateList; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.hep.HepRelVertex; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Exchange; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexTableInputRef; +import org.apache.calcite.util.BuiltInMethod; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.Util; + +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +/** + * Utility to extract Predicates that are present in the (sub)plan + * starting at this node. + * + * This should be used to infer whether same filters are applied on + * a given plan by materialized view rewriting rules. + * + * The output predicates might contain references to columns produced + * by TableScan operators ({@link RexTableInputRef}). In turn, each TableScan + * operator is identified uniquely by its qualified name and an identifier. + * + * If the provider cannot infer the lineage for any of the expressions + * contain in any of the predicates, it will return null. Observe that + * this is different from the empty list of predicates, which means that + * there are not predicates in the (sub)plan. + * + */ +public class RelMdAllPredicates + implements MetadataHandler<BuiltInMetadata.AllPredicates> { + public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider + .reflectiveSource(BuiltInMethod.ALL_PREDICATES.method, new RelMdAllPredicates()); + + public MetadataDef<BuiltInMetadata.AllPredicates> getDef() { + return BuiltInMetadata.AllPredicates.DEF; + } + + /** Catch-all implementation for + * {@link BuiltInMetadata.AllPredicates#getAllPredicates()}, + * invoked using reflection. + * + * @see org.apache.calcite.rel.metadata.RelMetadataQuery#getAllPredicates(RelNode) + */ + public RelOptPredicateList getAllPredicates(RelNode rel, RelMetadataQuery mq) { + return null; + } + + public RelOptPredicateList getAllPredicates(HepRelVertex rel, RelMetadataQuery mq) { + return mq.getAllPredicates(rel.getCurrentRel()); + } + + public RelOptPredicateList getAllPredicates(RelSubset rel, + RelMetadataQuery mq) { + return mq.getAllPredicates(Util.first(rel.getBest(), rel.getOriginal())); + } + + /** + * Extract predicates for a table scan. + */ + public RelOptPredicateList getAllPredicates(TableScan table, RelMetadataQuery mq) { + return RelOptPredicateList.EMPTY; + } + + /** + * Extract predicates for a project. + */ + public RelOptPredicateList getAllPredicates(Project project, RelMetadataQuery mq) { + return mq.getAllPredicates(project.getInput()); + } + + /** + * Add the Filter condition to the list obtained from the input. + */ + public RelOptPredicateList getAllPredicates(Filter filter, RelMetadataQuery mq) { + final RelNode input = filter.getInput(); + final RexBuilder rexBuilder = filter.getCluster().getRexBuilder(); + final RexNode pred = filter.getCondition(); + + final RelOptPredicateList predsBelow = mq.getAllPredicates(input); + if (predsBelow == null) { + // Safety check + return null; + } + + // Extract input fields referenced by Filter condition + final Set<RelDataTypeField> inputExtraFields = new LinkedHashSet<>(); + final RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(inputExtraFields); + pred.accept(inputFinder); + final ImmutableBitSet inputFieldsUsed = inputFinder.inputBitSet.build(); + + // Infer column origin expressions for given references + final Map<RexInputRef, Set<RexNode>> mapping = new LinkedHashMap<>(); + for (int idx : inputFieldsUsed) { + final RexInputRef ref = RexInputRef.of(idx, filter.getRowType().getFieldList()); + final Set<RexNode> originalExprs = mq.getExpressionLineage(filter, ref); + if (originalExprs == null) { + // Bail out + return null; + } + mapping.put(ref, originalExprs); + } + + // Replace with new expressions and return union of predicates + return predsBelow.union(rexBuilder, + RelOptPredicateList.of(rexBuilder, + RelMdExpressionLineage.createAllPossibleExpressions(rexBuilder, pred, mapping))); + } + + /** + * Add the Join condition to the list obtained from the input. + */ + public RelOptPredicateList getAllPredicates(Join join, RelMetadataQuery mq) { + if (join.getJoinType() != JoinRelType.INNER) { + // We cannot map origin of this expression. + return null; + } + + final RexBuilder rexBuilder = join.getCluster().getRexBuilder(); + final RexNode pred = join.getCondition(); + final RelNode leftInput = join.getLeft(); + final RelNode rightInput = join.getRight(); + final int nLeftColumns = leftInput.getRowType().getFieldList().size(); + + RelOptPredicateList newPreds = RelOptPredicateList.EMPTY; + for (RelNode input : join.getInputs()) { + final RelOptPredicateList inputPreds = mq.getAllPredicates(input); + if (inputPreds == null) { + // Bail out + return null; + } + newPreds = newPreds.union(rexBuilder, inputPreds); + } + + // Extract input fields referenced by Join condition + final Set<RelDataTypeField> inputExtraFields = new LinkedHashSet<>(); + final RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(inputExtraFields); + pred.accept(inputFinder); + final ImmutableBitSet inputFieldsUsed = inputFinder.inputBitSet.build(); + + // Infer column origin expressions for given references + final Map<RexInputRef, Set<RexNode>> mapping = new LinkedHashMap<>(); + for (int idx : inputFieldsUsed) { + if (idx < nLeftColumns) { + final RexInputRef inputRef = RexInputRef.of(idx, leftInput.getRowType().getFieldList()); + final Set<RexNode> originalExprs = mq.getExpressionLineage(leftInput, inputRef); + if (originalExprs == null) { + // Bail out + return null; + } + final RexInputRef ref = RexInputRef.of(idx, join.getRowType().getFieldList()); + mapping.put(ref, originalExprs); + } else { + // Right input. + final RexInputRef inputRef = RexInputRef.of(idx - nLeftColumns, + rightInput.getRowType().getFieldList()); + final Set<RexNode> originalExprs = mq.getExpressionLineage(rightInput, inputRef); + if (originalExprs == null) { + // Bail out + return null; + } + final RexInputRef ref = RexInputRef.of(idx, join.getRowType().getFieldList()); + mapping.put(ref, originalExprs); + } + } + + // Replace with new expressions and return union of predicates + return newPreds.union(rexBuilder, + RelOptPredicateList.of(rexBuilder, + RelMdExpressionLineage.createAllPossibleExpressions(rexBuilder, pred, mapping))); + } + + /** + * Extract predicates for an Aggregate. + */ + public RelOptPredicateList getAllPredicates(Aggregate agg, RelMetadataQuery mq) { + return mq.getAllPredicates(agg.getInput()); + } + + /** + * Extract predicates for a Union. + */ + public RelOptPredicateList getAllPredicates(Union union, RelMetadataQuery mq) { + final RexBuilder rexBuilder = union.getCluster().getRexBuilder(); + + RelOptPredicateList newPreds = RelOptPredicateList.EMPTY; + for (RelNode input : union.getInputs()) { + final RelOptPredicateList inputPreds = mq.getAllPredicates(input); + if (inputPreds == null) { + // Bail out + return null; + } + newPreds = newPreds.union(rexBuilder, inputPreds); + } + return newPreds; + } + + /** + * Extract predicates for a Sort. + */ + public RelOptPredicateList getAllPredicates(Sort sort, RelMetadataQuery mq) { + return mq.getAllPredicates(sort.getInput()); + } + + /** + * Extract predicates for an Exchange. + */ + public RelOptPredicateList getAllPredicates(Exchange exchange, + RelMetadataQuery mq) { + return mq.getAllPredicates(exchange.getInput()); + } + +} + +// End RelMdAllPredicates.java http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java new file mode 100644 index 0000000..5f9e5ba --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdExpressionLineage.java @@ -0,0 +1,450 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.metadata; + +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.hep.HepRelVertex; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Exchange; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexTableInputRef; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.util.BuiltInMethod; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.Util; + +import com.google.common.base.Function; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * RelMdExpressionLineage supplies a default implementation of + * {@link RelMetadataQuery#getExpressionLineage} for the standard logical algebra. + * + * The goal of this provider is to infer the lineage for the given expression. + * + * The output expressions might contain references to columns produced by TableScan + * operators ({@link RexTableInputRef}). In turn, each TableScan operator is identified + * uniquely by its qualified name and an identifier contained in . + * + * If the lineage cannot be inferred, we return null. + */ +public class RelMdExpressionLineage + implements MetadataHandler<BuiltInMetadata.ExpressionLineage> { + public static final RelMetadataProvider SOURCE = + ReflectiveRelMetadataProvider.reflectiveSource( + BuiltInMethod.EXPRESSION_LINEAGE.method, new RelMdExpressionLineage()); + + //~ Constructors ----------------------------------------------------------- + + private RelMdExpressionLineage() {} + + //~ Methods ---------------------------------------------------------------- + + public MetadataDef<BuiltInMetadata.ExpressionLineage> getDef() { + return BuiltInMetadata.ExpressionLineage.DEF; + } + + // Catch-all rule when none of the others apply. + public Set<RexNode> getExpressionLineage(RelNode rel, + RelMetadataQuery mq, RexNode outputExpression) { + return null; + } + + public Set<RexNode> getExpressionLineage(HepRelVertex rel, RelMetadataQuery mq, + RexNode outputExpression) { + return mq.getExpressionLineage(rel.getCurrentRel(), outputExpression); + } + + public Set<RexNode> getExpressionLineage(RelSubset rel, + RelMetadataQuery mq, RexNode outputExpression) { + return mq.getExpressionLineage(Util.first(rel.getBest(), rel.getOriginal()), + outputExpression); + } + + /** + * Expression lineage from TableScan. + * + * We extract the fields referenced by the expression and we express them + * using {@link RexTableInputRef}. + */ + public Set<RexNode> getExpressionLineage(TableScan rel, + RelMetadataQuery mq, RexNode outputExpression) { + final RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); + + // Extract input fields referenced by expression + final Set<RelDataTypeField> inputExtraFields = new LinkedHashSet<>(); + final RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(inputExtraFields); + outputExpression.accept(inputFinder); + final ImmutableBitSet inputFieldsUsed = inputFinder.inputBitSet.build(); + + // Infer column origin expressions for given references + final Map<RexInputRef, Set<RexNode>> mapping = new LinkedHashMap<>(); + for (int idx : inputFieldsUsed) { + final RexNode inputRef = RexTableInputRef.of( + new RelTableRef(rel.getTable().getQualifiedName().toString(), 0), + RexInputRef.of(idx, rel.getRowType().getFieldList())); + final Set<RexNode> originalExprs = Sets.newHashSet(inputRef); + final RexInputRef ref = RexInputRef.of(idx, rel.getRowType().getFieldList()); + mapping.put(ref, originalExprs); + } + + // Return result + return createAllPossibleExpressions(rexBuilder, outputExpression, mapping); + } + + /** + * Expression lineage from Aggregate. + * + * If the expression references grouping sets or aggregation function results, + * we cannot extract the lineage and we return null. + */ + public Set<RexNode> getExpressionLineage(Aggregate rel, + RelMetadataQuery mq, RexNode outputExpression) { + final RelNode input = rel.getInput(); + final RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); + + // Extract input fields referenced by expression + final Set<RelDataTypeField> inputExtraFields = new LinkedHashSet<>(); + final RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(inputExtraFields); + outputExpression.accept(inputFinder); + final ImmutableBitSet inputFieldsUsed = inputFinder.inputBitSet.build(); + + for (int idx : inputFieldsUsed) { + if (idx >= rel.getGroupCount()) { + // We cannot map origin of this expression. + return null; + } + } + + // Infer column origin expressions for given references + final Map<RexInputRef, Set<RexNode>> mapping = new LinkedHashMap<>(); + for (int idx : inputFieldsUsed) { + final RexInputRef inputRef = RexInputRef.of(rel.getGroupSet().nth(idx), + input.getRowType().getFieldList()); + final Set<RexNode> originalExprs = mq.getExpressionLineage(input, inputRef); + if (originalExprs == null) { + // Bail out + return null; + } + final RexInputRef ref = RexInputRef.of(idx, rel.getRowType().getFieldList()); + mapping.put(ref, originalExprs); + } + + // Return result + return createAllPossibleExpressions(rexBuilder, outputExpression, mapping); + } + + /** + * Expression lineage from Join. + * + * We only extract the lineage for INNER joins. + */ + public Set<RexNode> getExpressionLineage(Join rel, RelMetadataQuery mq, + RexNode outputExpression) { + if (rel.getJoinType() != JoinRelType.INNER) { + // We cannot map origin of this expression. + return null; + } + + final RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); + final RelNode leftInput = rel.getLeft(); + final RelNode rightInput = rel.getRight(); + final int nLeftColumns = leftInput.getRowType().getFieldList().size(); + + // Infer column origin expressions for given references + final Multimap<String, RelTableRef> qualifiedNamesToRefs = HashMultimap.create(); + final Map<RelTableRef, RelTableRef> currentTablesMapping = new HashMap<>(); + final Map<RexInputRef, Set<RexNode>> mapping = new LinkedHashMap<>(); + for (int idx = 0; idx < rel.getRowType().getFieldList().size(); idx++) { + if (idx < nLeftColumns) { + final RexInputRef inputRef = RexInputRef.of(idx, leftInput.getRowType().getFieldList()); + final Set<RexNode> originalExprs = mq.getExpressionLineage(leftInput, inputRef); + if (originalExprs == null) { + // Bail out + return null; + } + // Gather table references, left input references remain unchanged + final Set<RelTableRef> tableRefs = + RexUtil.gatherTableReferences(Lists.newArrayList(originalExprs)); + for (RelTableRef leftRef : tableRefs) { + qualifiedNamesToRefs.put(leftRef.getQualifiedName(), leftRef); + } + mapping.put(RexInputRef.of(idx, rel.getRowType().getFieldList()), originalExprs); + } else { + // Right input. + final RexInputRef inputRef = RexInputRef.of(idx - nLeftColumns, + rightInput.getRowType().getFieldList()); + final Set<RexNode> originalExprs = mq.getExpressionLineage(rightInput, inputRef); + if (originalExprs == null) { + // Bail out + return null; + } + // Gather table references, right input references might need to be + // updated if there are table names clashes with left input + final Set<RelTableRef> tableRefs = + RexUtil.gatherTableReferences(Lists.newArrayList(originalExprs)); + for (RelTableRef rightRef : tableRefs) { + int shift = 0; + Collection<RelTableRef> lRefs = qualifiedNamesToRefs.get( + rightRef.getQualifiedName()); + if (lRefs != null) { + shift = lRefs.size(); + } + currentTablesMapping.put(rightRef, + new RelTableRef(rightRef.getQualifiedName(), shift + rightRef.getIdentifier())); + } + final Set<RexNode> updatedExprs = Sets.newHashSet( + Iterables.transform( + originalExprs, + new Function<RexNode, RexNode>() { + @Override public RexNode apply(RexNode e) { + return RexUtil.swapTableReferences(rexBuilder, e, currentTablesMapping); + } + } + )); + mapping.put(RexInputRef.of(idx, rel.getRowType().getFieldList()), updatedExprs); + } + } + + // Return result + return createAllPossibleExpressions(rexBuilder, outputExpression, mapping); + } + + /** + * Expression lineage from Union. + * + * For Union operator, we might be able to extract multiple origins for the + * references in the given expression. + */ + public Set<RexNode> getExpressionLineage(Union rel, RelMetadataQuery mq, + RexNode outputExpression) { + final RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); + + // Infer column origin expressions for given references + final Multimap<String, RelTableRef> qualifiedNamesToRefs = HashMultimap.create(); + final Map<RexInputRef, Set<RexNode>> mapping = new LinkedHashMap<>(); + for (RelNode input : rel.getInputs()) { + final Map<RelTableRef, RelTableRef> currentTablesMapping = new HashMap<>(); + for (int idx = 0; idx < input.getRowType().getFieldList().size(); idx++) { + final RexInputRef inputRef = RexInputRef.of(idx, input.getRowType().getFieldList()); + final Set<RexNode> originalExprs = mq.getExpressionLineage(input, inputRef); + if (originalExprs == null) { + // Bail out + return null; + } + + final RexInputRef ref = RexInputRef.of(idx, rel.getRowType().getFieldList()); + // Gather table references, references might need to be + // updated + final Set<RelTableRef> tableRefs = + RexUtil.gatherTableReferences(Lists.newArrayList(originalExprs)); + for (RelTableRef tableRef : tableRefs) { + int shift = 0; + Collection<RelTableRef> lRefs = qualifiedNamesToRefs.get( + tableRef.getQualifiedName()); + if (lRefs != null) { + shift = lRefs.size(); + } + currentTablesMapping.put(tableRef, + new RelTableRef(tableRef.getQualifiedName(), shift + tableRef.getIdentifier())); + } + final Set<RexNode> updatedExprs = Sets.newHashSet( + Iterables.transform( + originalExprs, + new Function<RexNode, RexNode>() { + @Override public RexNode apply(RexNode e) { + return RexUtil.swapTableReferences(rexBuilder, e, currentTablesMapping); + } + } + )); + final Set<RexNode> set = mapping.get(ref); + if (set != null) { + set.addAll(updatedExprs); + } else { + mapping.put(ref, updatedExprs); + } + } + // Add to existing qualified names + for (RelTableRef newRef : currentTablesMapping.values()) { + qualifiedNamesToRefs.put(newRef.getQualifiedName(), newRef); + } + } + + // Return result + return createAllPossibleExpressions(rexBuilder, outputExpression, mapping); + } + + /** + * Expression lineage from Project. + */ + public Set<RexNode> getExpressionLineage(Project rel, + final RelMetadataQuery mq, RexNode outputExpression) { + final RelNode input = rel.getInput(); + final RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); + + // Extract input fields referenced by expression + final Set<RelDataTypeField> inputExtraFields = new LinkedHashSet<>(); + final RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(inputExtraFields); + outputExpression.accept(inputFinder); + final ImmutableBitSet inputFieldsUsed = inputFinder.inputBitSet.build(); + + // Infer column origin expressions for given references + final Map<RexInputRef, Set<RexNode>> mapping = new LinkedHashMap<>(); + for (int idx : inputFieldsUsed) { + final RexNode inputExpr = rel.getChildExps().get(idx); + final Set<RexNode> originalExprs = mq.getExpressionLineage(input, inputExpr); + if (originalExprs == null) { + // Bail out + return null; + } + final RexInputRef ref = RexInputRef.of(idx, rel.getRowType().getFieldList()); + mapping.put(ref, originalExprs); + } + + // Return result + return createAllPossibleExpressions(rexBuilder, outputExpression, mapping); + } + + /** + * Expression lineage from Filter. + */ + public Set<RexNode> getExpressionLineage(Filter rel, + RelMetadataQuery mq, RexNode outputExpression) { + return mq.getExpressionLineage(rel.getInput(), outputExpression); + } + + /** + * Expression lineage from Sort. + */ + public Set<RexNode> getExpressionLineage(Sort rel, RelMetadataQuery mq, + RexNode outputExpression) { + return mq.getExpressionLineage(rel.getInput(), outputExpression); + } + + /** + * Expression lineage from Exchange. + */ + public Set<RexNode> getExpressionLineage(Exchange rel, + RelMetadataQuery mq, RexNode outputExpression) { + return mq.getExpressionLineage(rel.getInput(), outputExpression); + } + + /** + * Given an expression, it will create all equivalent expressions resulting + * from replacing all possible combinations of references in the mapping by + * the corresponding expressions. + * + * @param rexBuilder rexBuilder + * @param expr expression + * @param mapping mapping + * @return set of resulting expressions equivalent to the input expression + */ + protected static Set<RexNode> createAllPossibleExpressions(RexBuilder rexBuilder, + RexNode expr, Map<RexInputRef, Set<RexNode>> mapping) { + // Extract input fields referenced by expression + final Set<RelDataTypeField> inputExtraFields = new LinkedHashSet<>(); + final RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(inputExtraFields); + expr.accept(inputFinder); + final ImmutableBitSet predFieldsUsed = inputFinder.inputBitSet.build(); + + return createAllPossibleExpressions(rexBuilder, expr, predFieldsUsed, mapping, + new HashMap<RexInputRef, RexNode>()); + } + + private static Set<RexNode> createAllPossibleExpressions(RexBuilder rexBuilder, + RexNode expr, ImmutableBitSet predFieldsUsed, Map<RexInputRef, Set<RexNode>> mapping, + Map<RexInputRef, RexNode> singleMapping) { + final RexInputRef inputRef = mapping.keySet().iterator().next(); + final Set<RexNode> replacements = mapping.remove(inputRef); + Set<RexNode> result = new HashSet<>(); + assert !replacements.isEmpty(); + if (predFieldsUsed.indexOf(inputRef.getIndex()) != -1) { + for (RexNode replacement : replacements) { + singleMapping.put(inputRef, replacement); + createExpressions(rexBuilder, expr, predFieldsUsed, mapping, singleMapping, result); + singleMapping.remove(inputRef); + } + } else { + createExpressions(rexBuilder, expr, predFieldsUsed, mapping, singleMapping, result); + } + mapping.put(inputRef, replacements); + return result; + } + + private static void createExpressions(RexBuilder rexBuilder, + RexNode expr, ImmutableBitSet predFieldsUsed, Map<RexInputRef, Set<RexNode>> mapping, + Map<RexInputRef, RexNode> singleMapping, Set<RexNode> result) { + if (mapping.isEmpty()) { + final RexReplacer replacer = new RexReplacer(singleMapping); + final List<RexNode> updatedPreds = new ArrayList<>( + RelOptUtil.conjunctions( + rexBuilder.copy(expr))); + replacer.mutate(updatedPreds); + result.addAll(updatedPreds); + } else { + result.addAll( + createAllPossibleExpressions( + rexBuilder, expr, predFieldsUsed, mapping, singleMapping)); + } + } + + /** + * Replaces expressions with their equivalences. Note that we only have to + * look for RexInputRef. + */ + private static class RexReplacer extends RexShuttle { + private final Map<RexInputRef, RexNode> replacementValues; + + RexReplacer(Map<RexInputRef, RexNode> replacementValues) { + this.replacementValues = replacementValues; + } + + @Override public RexNode visitInputRef(RexInputRef inputRef) { + return replacementValues.get(inputRef); + } + } + +} + +// End RelMdExpressionLineage.java http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rel/metadata/RelMdNodeTypes.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdNodeTypes.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdNodeTypes.java new file mode 100644 index 0000000..566df3a --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdNodeTypes.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.metadata; + +import org.apache.calcite.plan.hep.HepRelVertex; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Calc; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Intersect; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.Minus; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.SemiJoin; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.core.Values; +import org.apache.calcite.util.BuiltInMethod; +import org.apache.calcite.util.Util; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + +/** + * RelMdNodeTypeCount supplies a default implementation of + * {@link RelMetadataQuery#getNodeTypes} for the standard logical algebra. + */ +public class RelMdNodeTypes + implements MetadataHandler<BuiltInMetadata.NodeTypes> { + public static final RelMetadataProvider SOURCE = + ReflectiveRelMetadataProvider.reflectiveSource( + BuiltInMethod.NODE_TYPES.method, new RelMdNodeTypes()); + + //~ Methods ---------------------------------------------------------------- + + public MetadataDef<BuiltInMetadata.NodeTypes> getDef() { + return BuiltInMetadata.NodeTypes.DEF; + } + + /** Catch-all implementation for + * {@link BuiltInMetadata.NodeTypeCount#getNodeTypes()}, + * invoked using reflection. + * + * @see org.apache.calcite.rel.metadata.RelMetadataQuery#getNodeTypes(RelNode) + */ + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(RelNode rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, RelNode.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(HepRelVertex rel, + RelMetadataQuery mq) { + return mq.getNodeTypes(rel.getCurrentRel()); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(RelSubset rel, + RelMetadataQuery mq) { + return mq.getNodeTypes(Util.first(rel.getBest(), rel.getOriginal())); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(Union rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, Union.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(Intersect rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, Intersect.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(Minus rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, Minus.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(Filter rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, Filter.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(Calc rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, Calc.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(Project rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, Project.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(Sort rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, Sort.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(Join rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, Join.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(SemiJoin rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, SemiJoin.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(Aggregate rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, Aggregate.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(TableScan rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, TableScan.class, mq); + } + + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(Values rel, + RelMetadataQuery mq) { + return getNodeTypes(rel, Values.class, mq); + } + + private static Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(RelNode rel, + Class<? extends RelNode> c, RelMetadataQuery mq) { + final Multimap<Class<? extends RelNode>, RelNode> nodeTypeCount = ArrayListMultimap.create(); + for (RelNode input : rel.getInputs()) { + Multimap<Class<? extends RelNode>, RelNode> partialNodeTypeCount = + mq.getNodeTypes(input); + if (partialNodeTypeCount == null) { + return null; + } + nodeTypeCount.putAll(partialNodeTypeCount); + } + nodeTypeCount.put(c, rel); + return nodeTypeCount; + } + +} + +// End RelMdNodeTypes.java http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java index 7b63ac9..4ef5592 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUtil.java @@ -843,6 +843,7 @@ public class RelMdUtil { } return alreadySorted && alreadySmaller; } + } // End RelMdUtil.java http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java index 85f747c..9dd19dc 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java @@ -29,6 +29,7 @@ import org.apache.calcite.util.ImmutableBitSet; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.collect.Multimap; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; @@ -84,6 +85,7 @@ public class RelMetadataQuery { private BuiltInMetadata.Collation.Handler collationHandler; private BuiltInMetadata.ColumnOrigin.Handler columnOriginHandler; + private BuiltInMetadata.ExpressionLineage.Handler expressionLineageHandler; private BuiltInMetadata.ColumnUniqueness.Handler columnUniquenessHandler; private BuiltInMetadata.CumulativeCost.Handler cumulativeCostHandler; private BuiltInMetadata.DistinctRowCount.Handler distinctRowCountHandler; @@ -97,6 +99,8 @@ public class RelMetadataQuery { private BuiltInMetadata.PercentageOriginalRows.Handler percentageOriginalRowsHandler; private BuiltInMetadata.PopulationSize.Handler populationSizeHandler; private BuiltInMetadata.Predicates.Handler predicatesHandler; + private BuiltInMetadata.AllPredicates.Handler allPredicatesHandler; + private BuiltInMetadata.NodeTypes.Handler nodeTypesHandler; private BuiltInMetadata.RowCount.Handler rowCountHandler; private BuiltInMetadata.Selectivity.Handler selectivityHandler; private BuiltInMetadata.Size.Handler sizeHandler; @@ -114,6 +118,7 @@ public class RelMetadataQuery { this.metadataProvider = Preconditions.checkNotNull(metadataProvider); this.collationHandler = prototype.collationHandler; this.columnOriginHandler = prototype.columnOriginHandler; + this.expressionLineageHandler = prototype.expressionLineageHandler; this.columnUniquenessHandler = prototype.columnUniquenessHandler; this.cumulativeCostHandler = prototype.cumulativeCostHandler; this.distinctRowCountHandler = prototype.distinctRowCountHandler; @@ -127,6 +132,8 @@ public class RelMetadataQuery { this.percentageOriginalRowsHandler = prototype.percentageOriginalRowsHandler; this.populationSizeHandler = prototype.populationSizeHandler; this.predicatesHandler = prototype.predicatesHandler; + this.allPredicatesHandler = prototype.allPredicatesHandler; + this.nodeTypesHandler = prototype.nodeTypesHandler; this.rowCountHandler = prototype.rowCountHandler; this.selectivityHandler = prototype.selectivityHandler; this.sizeHandler = prototype.sizeHandler; @@ -162,6 +169,7 @@ public class RelMetadataQuery { this.metadataProvider = null; this.collationHandler = initialHandler(BuiltInMetadata.Collation.Handler.class); this.columnOriginHandler = initialHandler(BuiltInMetadata.ColumnOrigin.Handler.class); + this.expressionLineageHandler = initialHandler(BuiltInMetadata.ExpressionLineage.Handler.class); this.columnUniquenessHandler = initialHandler(BuiltInMetadata.ColumnUniqueness.Handler.class); this.cumulativeCostHandler = initialHandler(BuiltInMetadata.CumulativeCost.Handler.class); this.distinctRowCountHandler = initialHandler(BuiltInMetadata.DistinctRowCount.Handler.class); @@ -176,6 +184,8 @@ public class RelMetadataQuery { initialHandler(BuiltInMetadata.PercentageOriginalRows.Handler.class); this.populationSizeHandler = initialHandler(BuiltInMetadata.PopulationSize.Handler.class); this.predicatesHandler = initialHandler(BuiltInMetadata.Predicates.Handler.class); + this.allPredicatesHandler = initialHandler(BuiltInMetadata.AllPredicates.Handler.class); + this.nodeTypesHandler = initialHandler(BuiltInMetadata.NodeTypes.Handler.class); this.rowCountHandler = initialHandler(BuiltInMetadata.RowCount.Handler.class); this.selectivityHandler = initialHandler(BuiltInMetadata.Selectivity.Handler.class); this.sizeHandler = initialHandler(BuiltInMetadata.Size.Handler.class); @@ -191,6 +201,24 @@ public class RelMetadataQuery { /** * Returns the + * {@link BuiltInMetadata.NodeTypeCount#getNodeTypeCount()} + * statistic. + * + * @param rel the relational expression + * @return + */ + public Multimap<Class<? extends RelNode>, RelNode> getNodeTypes(RelNode rel) { + for (;;) { + try { + return nodeTypesHandler.getNodeTypes(rel, this); + } catch (JaninoRelMetadataProvider.NoHandler e) { + nodeTypesHandler = revise(e.relClass, BuiltInMetadata.NodeTypes.DEF); + } + } + } + + /** + * Returns the * {@link BuiltInMetadata.RowCount#getRowCount()} * statistic. * @@ -352,6 +380,20 @@ public class RelMetadataQuery { } /** + * Determines the origin of a column. + */ + public Set<RexNode> getExpressionLineage(RelNode rel, RexNode expression) { + for (;;) { + try { + return expressionLineageHandler.getExpressionLineage(rel, this, expression); + } catch (JaninoRelMetadataProvider.NoHandler e) { + expressionLineageHandler = + revise(e.relClass, BuiltInMetadata.ExpressionLineage.DEF); + } + } + } + + /** * Determines the origin of a {@link RelNode}, provided it maps to a single * table, optionally with filtering and projection. * @@ -749,6 +791,24 @@ public class RelMetadataQuery { /** * Returns the + * {@link BuiltInMetadata.AllPredicates#getAllPredicates()} + * statistic. + * + * @param rel the relational expression + * @return All predicates within and below this RelNode + */ + public RelOptPredicateList getAllPredicates(RelNode rel) { + for (;;) { + try { + return allPredicatesHandler.getAllPredicates(rel, this); + } catch (JaninoRelMetadataProvider.NoHandler e) { + allPredicatesHandler = revise(e.relClass, BuiltInMetadata.AllPredicates.DEF); + } + } + } + + /** + * Returns the * {@link BuiltInMetadata.ExplainVisibility#isVisibleInExplain(SqlExplainLevel)} * statistic. * http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rel/metadata/RelTableRef.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelTableRef.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelTableRef.java new file mode 100644 index 0000000..bdd032f --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelTableRef.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.metadata; + +/** + * + * + */ +public class RelTableRef { + + private final String qualifiedName; + private final int identifier; + private final String digest; + + public RelTableRef(String qualifiedName, int identifier) { + this.qualifiedName = qualifiedName; + this.identifier = identifier; + this.digest = qualifiedName + ".#" + identifier; + } + + //~ Methods ---------------------------------------------------------------- + + @Override public boolean equals(Object obj) { + return this == obj + || obj instanceof RelTableRef + && qualifiedName.equals(((RelTableRef) obj).qualifiedName) + && identifier == ((RelTableRef) obj).identifier; + } + + @Override public int hashCode() { + return digest.hashCode(); + } + + public String getQualifiedName() { + return qualifiedName; + } + + public int getIdentifier() { + return identifier; + } + + @Override public String toString() { + return digest; + } + +} + +// End RelTableRef.java http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java b/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java index a16a834..f632e94 100644 --- a/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java +++ b/core/src/main/java/org/apache/calcite/rex/LogicVisitor.java @@ -163,6 +163,10 @@ public class LogicVisitor implements RexBiVisitor<Logic, Logic> { return end(subQuery, arg); } + @Override public Logic visitTableInputRef(RexTableInputRef ref, Logic arg) { + return end(ref, arg); + } + @Override public Logic visitPatternFieldRef(RexPatternFieldRef ref, Logic arg) { return end(ref, arg); } http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java index 8e0b014..aa494c2 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitor.java @@ -48,6 +48,8 @@ public interface RexBiVisitor<R, P> { R visitSubQuery(RexSubQuery subQuery, P arg); + R visitTableInputRef(RexTableInputRef ref, P arg); + R visitPatternFieldRef(RexPatternFieldRef ref, P arg); } http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rex/RexShuttle.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java index b642503..6714be1 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexShuttle.java +++ b/core/src/main/java/org/apache/calcite/rex/RexShuttle.java @@ -89,6 +89,10 @@ public class RexShuttle implements RexVisitor<RexNode> { } } + @Override public RexNode visitTableInputRef(RexTableInputRef ref) { + return ref; + } + @Override public RexNode visitPatternFieldRef(RexPatternFieldRef fieldRef) { return fieldRef; } http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rex/RexSimplify.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java index a910778..fb791f5 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java +++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java @@ -29,9 +29,11 @@ import org.apache.calcite.util.Util; import com.google.common.base.Preconditions; import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.BoundType; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; +import com.google.common.collect.Range; import java.util.ArrayList; import java.util.Collection; @@ -563,8 +565,8 @@ public class RexSimplify { return simplify(terms.get(0)); } // Try to simplify the expression - final Multimap<String, Pair<String, RexNode>> equalityTerms = - ArrayListMultimap.create(); + final Multimap<String, Pair<String, RexNode>> equalityTerms = ArrayListMultimap.create(); + final Map<String, Pair<Range, List<RexNode>>> rangeTerms = new HashMap<>(); final Map<String, String> equalityConstantTerms = new HashMap<>(); final Set<String> negatedTerms = new HashSet<>(); final Set<String> nullOperands = new HashSet<>(); @@ -611,23 +613,23 @@ public class RexSimplify { RexCall rightCast = (RexCall) right; comparedOperands.add(rightCast.getOperands().get(0).toString()); } - // Check for equality on different constants. If the same ref or - // CAST(ref) is equal to different constants, this condition cannot be - // satisfied, and hence it can be evaluated to FALSE. + // Check for equality on different constants. If the same ref or CAST(ref) + // is equal to different constants, this condition cannot be satisfied, + // and hence it can be evaluated to FALSE + final boolean leftRef = RexUtil.isReferenceOrAccess(left, true); + final boolean rightRef = RexUtil.isReferenceOrAccess(right, true); + final boolean leftConstant = left.isA(SqlKind.LITERAL); + final boolean rightConstant = right.isA(SqlKind.LITERAL); if (term.getKind() == SqlKind.EQUALS) { - final boolean leftRef = RexUtil.isReferenceOrAccess(left, true); - final boolean rightRef = RexUtil.isReferenceOrAccess(right, true); - if (right instanceof RexLiteral && leftRef) { + if (leftRef && rightConstant) { final String literal = right.toString(); - final String prevLiteral = - equalityConstantTerms.put(left.toString(), literal); + final String prevLiteral = equalityConstantTerms.put(left.toString(), literal); if (prevLiteral != null && !literal.equals(prevLiteral)) { return rexBuilder.makeLiteral(false); } - } else if (left instanceof RexLiteral && rightRef) { + } else if (leftConstant && rightRef) { final String literal = left.toString(); - final String prevLiteral = - equalityConstantTerms.put(right.toString(), literal); + final String prevLiteral = equalityConstantTerms.put(right.toString(), literal); if (prevLiteral != null && !literal.equals(prevLiteral)) { return rexBuilder.makeLiteral(false); } @@ -637,19 +639,39 @@ public class RexSimplify { } // Assume the expression a > 5 is part of a Filter condition. // Then we can derive the negated term: a <= 5. - // But as comparison is string-based and thus operands order-dependent, + // But as the comparison is string based and thus operands order dependent, // we should also add the inverted negated term: 5 >= a. - // Observe that for creating the inverted term we invert the list of - // operands. + // Observe that for creating the inverted term we invert the list of operands. RexNode negatedTerm = RexUtil.negate(rexBuilder, call); if (negatedTerm != null) { negatedTerms.add(negatedTerm.toString()); - RexNode invertNegatedTerm = - RexUtil.invert(rexBuilder, (RexCall) negatedTerm); + RexNode invertNegatedTerm = RexUtil.invert(rexBuilder, (RexCall) negatedTerm); if (invertNegatedTerm != null) { negatedTerms.add(invertNegatedTerm.toString()); } } + // Range + SqlKind comparison = null; + RexNode ref = null; + RexLiteral constant = null; + if (leftRef && rightConstant) { + comparison = term.getKind(); + ref = left; + constant = (RexLiteral) right; + } else if (leftConstant && rightRef) { + comparison = term.getKind().reverse(); + constant = (RexLiteral) left; + ref = right; + } + if (comparison != null + && comparison != SqlKind.NOT_EQUALS) { // NOT_EQUALS not supported + final RexNode result = processRange(rexBuilder, terms, rangeTerms, + term, ref, constant, comparison); + if (result != null) { + // Not satisfiable + return result; + } + } break; case IN: comparedOperands.add(((RexCall) term).operands.get(0).toString()); @@ -717,8 +739,7 @@ public class RexSimplify { if (!RexUtil.isDeterministic(notDisjunction)) { continue; } - final List<String> terms2Set = - RexUtil.strings(RelOptUtil.conjunctions(notDisjunction)); + final List<String> terms2Set = RexUtil.strings(RelOptUtil.conjunctions(notDisjunction)); if (termsSet.containsAll(terms2Set)) { return rexBuilder.makeLiteral(false); } @@ -800,6 +821,219 @@ public class RexSimplify { } } + private static RexNode processRange(RexBuilder rexBuilder, + List<RexNode> terms, Map<String, Pair<Range, List<RexNode>>> rangeTerms, + RexNode term, RexNode ref, RexLiteral constant, SqlKind comparison) { + final Comparable v0 = constant.getValue(); + Pair<Range, List<RexNode>> p = rangeTerms.get(ref.toString()); + if (p == null) { + Range r; + switch (comparison) { + case EQUALS: + r = Range.singleton(v0); + break; + case LESS_THAN: + r = Range.lessThan(v0); + break; + case LESS_THAN_OR_EQUAL: + r = Range.atMost(v0); + break; + case GREATER_THAN: + r = Range.greaterThan(v0); + break; + case GREATER_THAN_OR_EQUAL: + r = Range.atLeast(v0); + break; + default: + throw new AssertionError(); + } + rangeTerms.put(ref.toString(), + new Pair(r, ImmutableList.of(term))); + } else { + // Exists + boolean removeUpperBound = false; + boolean removeLowerBound = false; + Range r = p.left; + switch (comparison) { + case EQUALS: + if (!r.contains(v0)) { + // Range is empty, not satisfiable + return rexBuilder.makeLiteral(false); + } + rangeTerms.put(ref.toString(), + new Pair(Range.singleton(v0), ImmutableList.of(term))); + // remove + terms.removeAll(p.right); + break; + case LESS_THAN: { + int comparisonResult = 0; + if (r.hasUpperBound()) { + comparisonResult = v0.compareTo(r.upperEndpoint()); + } + if (comparisonResult <= 0) { + // 1) No upper bound, or + // 2) We need to open the upper bound, or + // 3) New upper bound is lower than old upper bound + if (r.hasLowerBound()) { + if (v0.compareTo(r.lowerEndpoint()) < 0) { + // Range is empty, not satisfiable + return rexBuilder.makeLiteral(false); + } + // a <= x < b OR a < x < b + r = Range.range(r.lowerEndpoint(), r.lowerBoundType(), + v0, BoundType.OPEN); + } else { + // x < b + r = Range.lessThan(v0); + } + + if (r.isEmpty()) { + // Range is empty, not satisfiable + return rexBuilder.makeLiteral(false); + } + + // remove prev upper bound + removeUpperBound = true; + } else { + // Remove this term as it is contained in current upper bound + terms.remove(term); + } + break; + } + case LESS_THAN_OR_EQUAL: { + int comparisonResult = -1; + if (r.hasUpperBound()) { + comparisonResult = v0.compareTo(r.upperEndpoint()); + } + if (comparisonResult < 0) { + // 1) No upper bound, or + // 2) New upper bound is lower than old upper bound + if (r.hasLowerBound()) { + if (v0.compareTo(r.lowerEndpoint()) < 0) { + // Range is empty, not satisfiable + return rexBuilder.makeLiteral(false); + } + // a <= x <= b OR a < x <= b + r = Range.range(r.lowerEndpoint(), r.lowerBoundType(), + v0, BoundType.CLOSED); + } else { + // x <= b + r = Range.atMost(v0); + } + + if (r.isEmpty()) { + // Range is empty, not satisfiable + return rexBuilder.makeLiteral(false); + } + + // remove prev upper bound + removeUpperBound = true; + } else { + // Remove this term as it is contained in current upper bound + terms.remove(term); + } + break; + } + case GREATER_THAN: { + int comparisonResult = 0; + if (r.hasLowerBound()) { + comparisonResult = v0.compareTo(r.lowerEndpoint()); + } + if (comparisonResult >= 0) { + // 1) No lower bound, or + // 2) We need to open the lower bound, or + // 3) New lower bound is greater than old lower bound + if (r.hasUpperBound()) { + if (v0.compareTo(r.upperEndpoint()) > 0) { + // Range is empty, not satisfiable + return rexBuilder.makeLiteral(false); + } + // a < x <= b OR a < x < b + r = Range.range(v0, BoundType.OPEN, + r.upperEndpoint(), r.upperBoundType()); + } else { + // x > a + r = Range.greaterThan(v0); + } + + if (r.isEmpty()) { + // Range is empty, not satisfiable + return rexBuilder.makeLiteral(false); + } + + // remove prev lower bound + removeLowerBound = true; + } else { + // Remove this term as it is contained in current lower bound + terms.remove(term); + } + break; + } + case GREATER_THAN_OR_EQUAL: { + int comparisonResult = 1; + if (r.hasLowerBound()) { + comparisonResult = v0.compareTo(r.lowerEndpoint()); + } + if (comparisonResult > 0) { + // 1) No lower bound, or + // 2) New lower bound is greater than old lower bound + if (r.hasUpperBound()) { + if (v0.compareTo(r.upperEndpoint()) > 0) { + // Range is empty, not satisfiable + return rexBuilder.makeLiteral(false); + } + // a <= x <= b OR a <= x < b + r = Range.range(v0, BoundType.CLOSED, + r.upperEndpoint(), r.upperBoundType()); + } else { + // x >= a + r = Range.atLeast(v0); + } + + if (r.isEmpty()) { + // Range is empty, not satisfiable + return rexBuilder.makeLiteral(false); + } + + // remove prev lower bound + removeLowerBound = true; + } else { + // Remove this term as it is contained in current lower bound + terms.remove(term); + } + break; + } + default: + throw new AssertionError(); + } + if (removeUpperBound) { + ImmutableList.Builder<RexNode> newBounds = ImmutableList.builder(); + for (RexNode e : p.right) { + if (e.isA(SqlKind.LESS_THAN) || e.isA(SqlKind.LESS_THAN_OR_EQUAL)) { + terms.remove(e); + } else { + newBounds.add(e); + } + } + newBounds.add(term); + rangeTerms.put(ref.toString(), new Pair(r, newBounds.build())); + } else if (removeLowerBound) { + ImmutableList.Builder<RexNode> newBounds = ImmutableList.builder(); + for (RexNode e : p.right) { + if (e.isA(SqlKind.GREATER_THAN) || e.isA(SqlKind.GREATER_THAN_OR_EQUAL)) { + terms.remove(e); + } else { + newBounds.add(e); + } + } + newBounds.add(term); + rangeTerms.put(ref.toString(), new Pair(r, newBounds.build())); + } + } + // Default + return null; + } + } // End RexSimplify.java http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java b/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java new file mode 100644 index 0000000..7e2d26b --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rex/RexTableInputRef.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rex; + +import org.apache.calcite.rel.metadata.RelTableRef; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlKind; + +/** + * Variable which references a field of an input relational expression + */ +public class RexTableInputRef extends RexInputRef { + + private final RelTableRef tableRef; + + public RexTableInputRef(RelTableRef tableRef, int index, RelDataType type) { + super(index, type); + this.tableRef = tableRef; + this.digest = tableRef.toString() + ".$" + index; + } + + //~ Methods ---------------------------------------------------------------- + + @Override public boolean equals(Object obj) { + return this == obj + || obj instanceof RexTableInputRef + && tableRef.equals(((RexTableInputRef) obj).tableRef) + && index == ((RexTableInputRef) obj).index; + } + + @Override public int hashCode() { + return digest.hashCode(); + } + + public RelTableRef getTableRef() { + return tableRef; + } + + public String getQualifiedName() { + return tableRef.getQualifiedName(); + } + + public int getIdentifier() { + return tableRef.getIdentifier(); + } + + public static RexTableInputRef of(RelTableRef tableRef, int index, RelDataType type) { + return new RexTableInputRef(tableRef, index, type); + } + + public static RexTableInputRef of(RelTableRef tableRef, RexInputRef ref) { + return new RexTableInputRef(tableRef, ref.getIndex(), ref.getType()); + } + + @Override public <R> R accept(RexVisitor<R> visitor) { + return visitor.visitTableInputRef(this); + } + + @Override public <R, P> R accept(RexBiVisitor<R, P> visitor, P arg) { + return visitor.visitTableInputRef(this, arg); + } + + @Override public SqlKind getKind() { + return SqlKind.TABLE_INPUT_REF; + } +} + +// End RexTableInputRef.java http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rex/RexUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java index f174f41..0af3fc7 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java +++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java @@ -24,6 +24,7 @@ import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.metadata.RelTableRef; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeFamily; @@ -490,6 +491,10 @@ public class RexUtil { return false; } + @Override public Boolean visitTableInputRef(RexTableInputRef ref) { + return false; + } + @Override public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) { return false; } @@ -821,6 +826,28 @@ public class RexUtil { return false; } + /** + * Returns whether a given tree contains any {link RexTableInputRef} nodes. + * + * @param node a RexNode tree + * @return first such node found or null if it there is no such node + */ + public static RexTableInputRef containsTableInputRef(RexNode node) { + try { + RexVisitor<Void> visitor = + new RexVisitorImpl<Void>(true) { + public Void visitTableInputRef(RexTableInputRef inputRef) { + throw new Util.FoundOne(inputRef); + } + }; + node.accept(visitor); + return null; + } catch (Util.FoundOne e) { + Util.swallow(e, null); + return (RexTableInputRef) e.getNode(); + } + } + public static boolean isAtomic(RexNode expr) { return (expr instanceof RexLiteral) || (expr instanceof RexVariable); } @@ -1840,6 +1867,61 @@ public class RexUtil { } } + public static RexNode swapTableReferences(final RexBuilder rexBuilder, + final RexNode node, final Map<RelTableRef, RelTableRef> tableMapping) { + return swapTableColumnReferences(rexBuilder, node, tableMapping, null); + } + + public static RexNode swapColumnReferences(final RexBuilder rexBuilder, + final RexNode node, final Map<RexTableInputRef, Set<RexTableInputRef>> ec) { + return swapTableColumnReferences(rexBuilder, node, null, ec); + } + + public static RexNode swapTableColumnReferences(final RexBuilder rexBuilder, + final RexNode node, final Map<RelTableRef, RelTableRef> tableMapping, + final Map<RexTableInputRef, Set<RexTableInputRef>> ec) { + RexShuttle visitor = + new RexShuttle() { + @Override public RexNode visitTableInputRef(RexTableInputRef inputRef) { + if (tableMapping != null) { + inputRef = new RexTableInputRef( + tableMapping.get(inputRef.getTableRef()), + inputRef.getIndex(), + inputRef.getType()); + } + if (ec != null) { + Set<RexTableInputRef> s = ec.get(inputRef); + if (s != null) { + inputRef = s.iterator().next(); + } + } + return inputRef; + } + }; + return visitor.apply(node); + } + + /** + * Gather all table references in input expressions. + * + * @param nodes expressions + * @return set of table references + */ + public static Set<RelTableRef> gatherTableReferences(final List<RexNode> nodes) { + final Set<RelTableRef> occurrences = new HashSet<>(); + RexVisitor<Void> visitor = + new RexVisitorImpl<Void>(true) { + @Override public Void visitTableInputRef(RexTableInputRef ref) { + occurrences.add(ref.getTableRef()); + return super.visitTableInputRef(ref); + } + }; + for (RexNode e : nodes) { + e.accept(visitor); + } + return occurrences; + } + //~ Inner Classes ---------------------------------------------------------- /** http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rex/RexVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitor.java b/core/src/main/java/org/apache/calcite/rex/RexVisitor.java index bf6dd99..a2a01d0 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexVisitor.java +++ b/core/src/main/java/org/apache/calcite/rex/RexVisitor.java @@ -48,6 +48,8 @@ public interface RexVisitor<R> { R visitSubQuery(RexSubQuery subQuery); + R visitTableInputRef(RexTableInputRef fieldRef); + R visitPatternFieldRef(RexPatternFieldRef fieldRef); } http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java index 4710f98..6e40a67 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java @@ -110,6 +110,10 @@ public class RexVisitorImpl<R> implements RexVisitor<R> { return r; } + @Override public R visitTableInputRef(RexTableInputRef ref) { + return null; + } + @Override public R visitPatternFieldRef(RexPatternFieldRef fieldRef) { return null; } http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/sql/SqlKind.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/sql/SqlKind.java b/core/src/main/java/org/apache/calcite/sql/SqlKind.java index 7a33772..07164b3 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlKind.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlKind.java @@ -561,6 +561,13 @@ public enum SqlKind { INPUT_REF, /** + * Reference to an input field, with a qualified name and an identifier + * + * <p>(Only used at the RexNode level.)</p> + */ + TABLE_INPUT_REF, + + /** * Reference to an input field, with pattern var as modifier * * <p>(Only used at the RexNode level.)</p> http://git-wip-us.apache.org/repos/asf/calcite/blob/41b05d78/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index f5161b2..df2f2f4 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -43,6 +43,7 @@ import org.apache.calcite.linq4j.function.Predicate2; import org.apache.calcite.linq4j.tree.FunctionExpression; import org.apache.calcite.linq4j.tree.Primitive; import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.metadata.BuiltInMetadata.AllPredicates; import org.apache.calcite.rel.metadata.BuiltInMetadata.Collation; import org.apache.calcite.rel.metadata.BuiltInMetadata.ColumnOrigin; import org.apache.calcite.rel.metadata.BuiltInMetadata.ColumnUniqueness; @@ -50,9 +51,11 @@ import org.apache.calcite.rel.metadata.BuiltInMetadata.CumulativeCost; import org.apache.calcite.rel.metadata.BuiltInMetadata.DistinctRowCount; import org.apache.calcite.rel.metadata.BuiltInMetadata.Distribution; import org.apache.calcite.rel.metadata.BuiltInMetadata.ExplainVisibility; +import org.apache.calcite.rel.metadata.BuiltInMetadata.ExpressionLineage; import org.apache.calcite.rel.metadata.BuiltInMetadata.MaxRowCount; import org.apache.calcite.rel.metadata.BuiltInMetadata.Memory; import org.apache.calcite.rel.metadata.BuiltInMetadata.MinRowCount; +import org.apache.calcite.rel.metadata.BuiltInMetadata.NodeTypes; import org.apache.calcite.rel.metadata.BuiltInMetadata.NonCumulativeCost; import org.apache.calcite.rel.metadata.BuiltInMetadata.Parallelism; import org.apache.calcite.rel.metadata.BuiltInMetadata.PercentageOriginalRows; @@ -361,6 +364,7 @@ public enum BuiltInMethod { ImmutableBitSet.class, boolean.class), COLLATIONS(Collation.class, "collations"), DISTRIBUTION(Distribution.class, "distribution"), + NODE_TYPES(NodeTypes.class, "getNodeTypes"), ROW_COUNT(RowCount.class, "getRowCount"), MAX_ROW_COUNT(MaxRowCount.class, "getMaxRowCount"), MIN_ROW_COUNT(MinRowCount.class, "getMinRowCount"), @@ -371,9 +375,11 @@ public enum BuiltInMethod { POPULATION_SIZE(PopulationSize.class, "getPopulationSize", ImmutableBitSet.class), COLUMN_ORIGIN(ColumnOrigin.class, "getColumnOrigins", int.class), + EXPRESSION_LINEAGE(ExpressionLineage.class, "getExpressionLineage", RexNode.class), CUMULATIVE_COST(CumulativeCost.class, "getCumulativeCost"), NON_CUMULATIVE_COST(NonCumulativeCost.class, "getNonCumulativeCost"), PREDICATES(Predicates.class, "getPredicates"), + ALL_PREDICATES(AllPredicates.class, "getAllPredicates"), EXPLAIN_VISIBILITY(ExplainVisibility.class, "isVisibleInExplain", SqlExplainLevel.class), SCALAR_EXECUTE1(Scalar.class, "execute", Context.class),
