Repository: calcite Updated Branches: refs/heads/master 478de569c -> e9d0ca673
http://git-wip-us.apache.org/repos/asf/calcite/blob/84b49f5b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index dbc230a..af25b30 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -153,7 +153,7 @@ public class RelMetadataTest extends SqlToRelTestBase { private static final double DEPT_SIZE = 4d; - private static final String EMP_QNAME = "[CATALOG, SALES, EMP]"; + private static final List<String> EMP_QNAME = ImmutableList.of("CATALOG", "SALES", "EMP"); //~ Methods ---------------------------------------------------------------- @@ -1513,7 +1513,7 @@ public class RelMetadataTest extends SqlToRelTestBase { final String inputRef = RexInputRef.of(4, tableRel.getRowType().getFieldList()).toString(); assertThat(r.size(), is(1)); final String resultString = r.iterator().next().toString(); - assertThat(resultString, startsWith(EMP_QNAME)); + assertThat(resultString, startsWith(EMP_QNAME.toString())); assertThat(resultString, endsWith(inputRef)); } @@ -1527,14 +1527,14 @@ public class RelMetadataTest extends SqlToRelTestBase { final Set<RexNode> r1 = mq.getExpressionLineage(rel, ref1); assertThat(r1.size(), is(1)); final RexTableInputRef result1 = (RexTableInputRef) r1.iterator().next(); - assertThat(result1.getQualifiedName(), is(EMP_QNAME)); + assertTrue(result1.getQualifiedName().equals(EMP_QNAME)); assertThat(result1.getIndex(), is(3)); final RexNode ref2 = RexInputRef.of(1, rel.getRowType().getFieldList()); final Set<RexNode> r2 = mq.getExpressionLineage(rel, ref2); assertThat(r2.size(), is(1)); final RexTableInputRef result2 = (RexTableInputRef) r2.iterator().next(); - assertThat(result2.getQualifiedName(), is(EMP_QNAME)); + assertTrue(result2.getQualifiedName().equals(EMP_QNAME)); assertThat(result2.getIndex(), is(7)); assertThat(result1.getIdentifier(), is(result2.getIdentifier())); @@ -1550,14 +1550,14 @@ public class RelMetadataTest extends SqlToRelTestBase { final Set<RexNode> r1 = mq.getExpressionLineage(rel, ref1); assertThat(r1.size(), is(1)); final RexTableInputRef result1 = (RexTableInputRef) r1.iterator().next(); - assertThat(result1.getQualifiedName(), is(EMP_QNAME)); + assertTrue(result1.getQualifiedName().equals(EMP_QNAME)); assertThat(result1.getIndex(), is(7)); final RexNode ref2 = RexInputRef.of(1, rel.getRowType().getFieldList()); final Set<RexNode> r2 = mq.getExpressionLineage(rel, ref2); assertThat(r2.size(), is(1)); final RexTableInputRef result2 = (RexTableInputRef) r2.iterator().next(); - assertThat(result2.getQualifiedName(), is(EMP_QNAME)); + assertTrue(result2.getQualifiedName().equals(EMP_QNAME)); assertThat(result2.getIndex(), is(3)); assertThat(result1.getIdentifier(), is(result2.getIdentifier())); @@ -1578,10 +1578,10 @@ public class RelMetadataTest extends SqlToRelTestBase { final RexCall call = (RexCall) result; assertThat(call.getOperands().size(), is(2)); final RexTableInputRef inputRef1 = (RexTableInputRef) call.getOperands().get(0); - assertThat(inputRef1.getQualifiedName(), is(EMP_QNAME)); + assertTrue(inputRef1.getQualifiedName().equals(EMP_QNAME)); assertThat(inputRef1.getIndex(), is(0)); final RexTableInputRef inputRef2 = (RexTableInputRef) call.getOperands().get(1); - assertThat(inputRef2.getQualifiedName(), is(EMP_QNAME)); + assertTrue(inputRef2.getQualifiedName().equals(EMP_QNAME)); assertThat(inputRef2.getIndex(), is(7)); assertThat(inputRef1.getIdentifier(), is(inputRef2.getIdentifier())); } @@ -1595,7 +1595,7 @@ public class RelMetadataTest extends SqlToRelTestBase { final Set<RexNode> r = mq.getExpressionLineage(rel, ref); assertThat(r.size(), is(1)); final RexTableInputRef result = (RexTableInputRef) r.iterator().next(); - assertThat(result.getQualifiedName(), is(EMP_QNAME)); + assertTrue(result.getQualifiedName().equals(EMP_QNAME)); assertThat(result.getIndex(), is(1)); } @@ -1608,7 +1608,7 @@ public class RelMetadataTest extends SqlToRelTestBase { final Set<RexNode> r = mq.getExpressionLineage(rel, ref); assertThat(r.size(), is(1)); final RexTableInputRef result = (RexTableInputRef) r.iterator().next(); - assertThat(result.getQualifiedName(), is("[CATALOG, SALES, BONUS]")); + assertTrue(result.getQualifiedName().equals(ImmutableList.of("CATALOG", "SALES", "BONUS"))); assertThat(result.getIndex(), is(0)); } @@ -1626,7 +1626,7 @@ public class RelMetadataTest extends SqlToRelTestBase { final String inputRef1 = RexInputRef.of(7, tableRel.getRowType().getFieldList()).toString(); assertThat(r1.size(), is(1)); final String resultString1 = r1.iterator().next().toString(); - assertThat(resultString1, startsWith(EMP_QNAME)); + assertThat(resultString1, startsWith(EMP_QNAME.toString())); assertThat(resultString1, endsWith(inputRef1)); final RexNode ref2 = RexInputRef.of(1, rel.getRowType().getFieldList()); @@ -1634,7 +1634,7 @@ public class RelMetadataTest extends SqlToRelTestBase { final String inputRef2 = RexInputRef.of(5, tableRel.getRowType().getFieldList()).toString(); assertThat(r2.size(), is(1)); final String resultString2 = r2.iterator().next().toString(); - assertThat(resultString2, startsWith(EMP_QNAME)); + assertThat(resultString2, startsWith(EMP_QNAME.toString())); assertThat(resultString2, endsWith(inputRef2)); assertThat(((RexTableInputRef) r1.iterator().next()).getIdentifier(), @@ -1663,7 +1663,7 @@ public class RelMetadataTest extends SqlToRelTestBase { final String inputRef = RexInputRef.of(1, tableRel.getRowType().getFieldList()).toString(); assertThat(r.size(), is(1)); final String resultString = r.iterator().next().toString(); - assertThat(resultString, startsWith(EMP_QNAME)); + assertThat(resultString, startsWith(EMP_QNAME.toString())); assertThat(resultString, endsWith(inputRef)); } @@ -1679,7 +1679,7 @@ public class RelMetadataTest extends SqlToRelTestBase { final String inputRef = RexInputRef.of(7, tableRel.getRowType().getFieldList()).toString(); assertThat(r.size(), is(1)); final String resultString = r.iterator().next().toString(); - assertThat(resultString, startsWith(EMP_QNAME)); + assertThat(resultString, startsWith(EMP_QNAME.toString())); assertThat(resultString, endsWith(inputRef)); } @@ -1708,7 +1708,7 @@ public class RelMetadataTest extends SqlToRelTestBase { assertThat(r.size(), is(2)); for (RexNode result : r) { final String resultString = result.toString(); - assertThat(resultString, startsWith(EMP_QNAME)); + assertThat(resultString, startsWith(EMP_QNAME.toString())); assertThat(resultString, endsWith(inputRef)); } @@ -1733,19 +1733,19 @@ public class RelMetadataTest extends SqlToRelTestBase { // With the union, we should get two origins // The first one should be the same one: join // The second should come from each union input - final Set<String> set = new HashSet<>(); + final Set<List<String>> set = new HashSet<>(); assertThat(r.size(), is(2)); for (RexNode result : r) { assertThat(result.getKind(), is(SqlKind.PLUS)); final RexCall call = (RexCall) result; assertThat(call.getOperands().size(), is(2)); final RexTableInputRef inputRef1 = (RexTableInputRef) call.getOperands().get(0); - assertThat(inputRef1.getQualifiedName(), is(EMP_QNAME)); + assertTrue(inputRef1.getQualifiedName().equals(EMP_QNAME)); // Add join alpha to set set.add(inputRef1.getQualifiedName()); assertThat(inputRef1.getIndex(), is(0)); final RexTableInputRef inputRef2 = (RexTableInputRef) call.getOperands().get(1); - assertThat(inputRef2.getQualifiedName(), is(EMP_QNAME)); + assertTrue(inputRef2.getQualifiedName().equals(EMP_QNAME)); assertThat(inputRef2.getIndex(), is(5)); assertThat(inputRef1.getIdentifier(), not(inputRef2.getIdentifier())); } @@ -1800,7 +1800,7 @@ public class RelMetadataTest extends SqlToRelTestBase { RexCall call = (RexCall) predicates.pulledUpPredicates.get(0); assertThat(call.getOperands().size(), is(2)); RexTableInputRef inputRef1 = (RexTableInputRef) call.getOperands().get(0); - assertThat(inputRef1.getQualifiedName(), is(EMP_QNAME)); + assertTrue(inputRef1.getQualifiedName().equals(EMP_QNAME)); assertThat(inputRef1.getIndex(), is(0)); final LogicalTableScan deptScan = @@ -1819,16 +1819,16 @@ public class RelMetadataTest extends SqlToRelTestBase { call = (RexCall) predicates.pulledUpPredicates.get(0); assertThat(call.getOperands().size(), is(2)); inputRef1 = (RexTableInputRef) call.getOperands().get(0); - assertThat(inputRef1.getQualifiedName(), is(EMP_QNAME)); + assertTrue(inputRef1.getQualifiedName().equals(EMP_QNAME)); assertThat(inputRef1.getIndex(), is(0)); // From Join call = (RexCall) predicates.pulledUpPredicates.get(1); assertThat(call.getOperands().size(), is(2)); inputRef1 = (RexTableInputRef) call.getOperands().get(0); - assertThat(inputRef1.getQualifiedName(), is(EMP_QNAME)); + assertTrue(inputRef1.getQualifiedName().equals(EMP_QNAME)); assertThat(inputRef1.getIndex(), is(7)); RexTableInputRef inputRef2 = (RexTableInputRef) call.getOperands().get(1); - assertThat(inputRef2.getQualifiedName(), is("[CATALOG, SALES, DEPT]")); + assertTrue(inputRef2.getQualifiedName().equals(ImmutableList.of("CATALOG", "SALES", "DEPT"))); assertThat(inputRef2.getIndex(), is(0)); } @@ -1844,7 +1844,7 @@ public class RelMetadataTest extends SqlToRelTestBase { RexCall call = (RexCall) pulledUpPredicates.get(0); assertThat(call.getOperands().size(), is(2)); final RexTableInputRef inputRef1 = (RexTableInputRef) call.getOperands().get(0); - assertThat(inputRef1.getQualifiedName(), is(EMP_QNAME)); + assertTrue(inputRef1.getQualifiedName().equals(EMP_QNAME)); assertThat(inputRef1.getIndex(), is(0)); final RexLiteral constant = (RexLiteral) call.getOperands().get(1); assertThat(constant.toString(), is("5")); @@ -1863,7 +1863,7 @@ public class RelMetadataTest extends SqlToRelTestBase { RexCall call = (RexCall) pulledUpPredicates.get(0); assertThat(call.getOperands().size(), is(2)); final RexTableInputRef inputRef1 = (RexTableInputRef) call.getOperands().get(0); - assertThat(inputRef1.getQualifiedName(), is(EMP_QNAME)); + assertTrue(inputRef1.getQualifiedName().equals(EMP_QNAME)); assertThat(inputRef1.getIndex(), is(0)); final RexLiteral constant = (RexLiteral) call.getOperands().get(1); assertThat(constant.toString(), is("5")); http://git-wip-us.apache.org/repos/asf/calcite/blob/84b49f5b/site/_docs/materialized_views.md ---------------------------------------------------------------------- diff --git a/site/_docs/materialized_views.md b/site/_docs/materialized_views.md index bbce64b..9856a87 100644 --- a/site/_docs/materialized_views.md +++ b/site/_docs/materialized_views.md @@ -33,9 +33,10 @@ For details, see the [lattices documentation]({{ site.baseurl }}/docs/lattice.ht ## Expose materialized views from adapters -Some adapters have their own notion of materialized views. +Some adapters and projects that rely on Calcite have their own notion of materialized views. For example, Apache Cassandra allows the user to define materialized views based on existing tables which are automatically maintained. The Cassandra adapter automatically exposes these materialized views to Calcite. +Another example is Apache Hive, whose integration with Calcite materialized views is ongoing. By understanding some tables as materialized views, Calcite has the opportunity to automatically rewrite queries to use these views. ## View-based query rewriting @@ -53,18 +54,34 @@ The following example is taken from the documentation of {SubstitutionVisitor}: * Result: `SELECT a, c FROM mv WHERE b = 4` Note that {result} uses the materialized view table {mv} and a simplified condition {b = 4}. -This can accomplish a large number of rewritings, but only those based on star schemas. -This type of rewriting cannot be used for more complex views. -{MaterializedViewJoinRule} attempts to match queries to views defined using arbitrary queries. -The logic of the rule is based on [this paper](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.95.113). +This can accomplish a large number of rewritings. +However, this approach is not scalable in the presence of complex views, e.g., views containing many join operators, +since it relies on the planner rules to create the equivalence between expressions. -There are several limitations to the current implementation: +In turn, two alternative rules that attempt to match queries to views defined using arbitrary queries, +have been proposed. They are both based on the ideas of the [same paper](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.95.113). -* The query defining the view must use only inner joins -* Only equality predicates are supported -* Predicates on tables used in the view must exactly match predicates in the query -* Rewriting is unoptimized and will attempt to match all views against each query +__MaterializedViewJoinRule__ is the first alternative. There are several limitations to the current implementation: + +1. The query defining the view must use only inner joins +2. Only equality predicates are supported +3. Predicates on tables used in the view must exactly match predicates in the query +4. Rewriting is unoptimized and will attempt to match all views against each query These limitations are not fundamental the approach however and will hopefully be removed in the future. Note that the rule is currently disabled by default. To make use of the rule, {MaterializedViewJoinRule.INSTANCE_PROJECT} and {MaterializedViewJoinRule.INSTANCE_TABLE_SCAN} need to be added to the planner. + +__AbstractMaterializedViewRule__ is the second alternative. It builds on the same ideas but it attempts to be more generic. +In particular, some of the limitations of the previous rule, such as number `2.` and `3.`, do not exist for this rule. +Additionally, the rule will be able to rewrite expressions rooted at an Aggregate operator, rolling aggregations up if necessary. + +However, this rule still presents some limitations too. In addition to `1.` and `4.` above, the rule presents following +shortcomings that we plan to address with follow-up extensions: + +* It does not produce rewritings using Union operators, e.g., a given query could be partially answered from the +{mv} (year = 2014) and from the query (not(year=2014)). This can be useful if {mv} is stored in a system such as +Druid. +* Currently query and {mv} must use the same tables. + +This rule is currently enabled by default.
