>From Ali Alsuliman <[email protected]>:
Ali Alsuliman has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21163?usp=email )
Change subject: meta fun in subplan
......................................................................
meta fun in subplan
Change-Id: I25b7972b6f4a7602cdecdfa6f055bf7ad41b51c7
---
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
A
asterixdb/asterix-app/src/test/resources/optimizerts/queries/meta/meta_in_subplan_1.sqlpp
A
asterixdb/asterix-app/src/test/resources/optimizerts/results/meta/meta_in_subplan_1.plan
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.4.query.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/results/meta/meta_after_gby/meta_after_gby.4.adm
M asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
8 files changed, 144 insertions(+), 9 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/63/21163/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
index 036c456..ef49276 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
@@ -21,7 +21,9 @@
import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -50,6 +52,7 @@
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import
org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import org.apache.hyracks.api.exceptions.SourceLocation;
@@ -69,12 +72,12 @@
return false;
}
hasApplied = true;
- visit(opRef);
+ visit(opRef, new HashSet<>(), true);
return rewritten;
}
- private ILogicalExpressionReferenceTransformWithCondition
visit(Mutable<ILogicalOperator> opRef)
- throws AlgebricksException {
+ private ILogicalExpressionReferenceTransformWithCondition
visit(Mutable<ILogicalOperator> opRef,
+ Set<ILogicalExpressionReferenceTransformWithCondition>
outerPlanTransformers, boolean topPlan) throws AlgebricksException {
ILogicalOperator op = opRef.getValue();
// reaches NTS or ETS.
@@ -84,7 +87,7 @@
// datascan returns a useful transform if the meta part is present in
the dataset.
if (op.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
DataSourceScanOperator scanOp = (DataSourceScanOperator) op;
- ILogicalExpressionReferenceTransformWithCondition inputTransformer
= visit(op.getInputs().get(0));
+ ILogicalExpressionReferenceTransformWithCondition inputTransformer
= visit(op.getInputs().get(0), outerPlanTransformers, topPlan);
DataSource dataSource = (DataSource) scanOp.getDataSource();
List<ILogicalExpressionReferenceTransformWithCondition>
transformers = null;
List<LogicalVariable> allVars = scanOp.getVariables();
@@ -136,7 +139,7 @@
// visit children in the depth-first order.
List<ILogicalExpressionReferenceTransformWithCondition> transformers =
new ArrayList<>();
for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
- ILogicalExpressionReferenceTransformWithCondition transformer =
visit(childRef);
+ ILogicalExpressionReferenceTransformWithCondition transformer =
visit(childRef, outerPlanTransformers, topPlan);
if
(!transformer.equals(NoOpExpressionReferenceTransform.INSTANCE)) {
transformers.add(transformer);
}
@@ -154,11 +157,33 @@
currentTransformer = new
CompositeExpressionReferenceTransform(transformers);
}
+ if (topPlan &&
!currentTransformer.equals(NoOpExpressionReferenceTransform.INSTANCE)) {
+ outerPlanTransformers.add(currentTransformer);
+ }
if (((AbstractLogicalOperator) op).hasNestedPlans()) {
AbstractOperatorWithNestedPlans opWithNestedPlans =
(AbstractOperatorWithNestedPlans) op;
for (ILogicalPlan nestedPlan : opWithNestedPlans.getNestedPlans())
{
for (Mutable<ILogicalOperator> root : nestedPlan.getRoots()) {
- visit(root);
+ visit(root, outerPlanTransformers, false);
+ }
+ }
+ }
+ if (!topPlan) {
+ // this is currently to handle a special case for operators with
nested plans (e.g. subplan) that may
+ // contain meta() in their nested plans. The transformer from the
outer plan will be pushed into the nested
+ // plan to rewrite meta() in the nested plan. The checks could be
relaxed for a broader use if we want to
+ // allow more cases of pushing down the transformer. The current
checks ensure that the transformer is only
+ // pushed into the nested plan when the nested plan has meta() and
there is only one transformer from the
+ // outer plan (i.e. there is only one data source with meta in the
outer plan) to avoid ambiguity.
+ if
(NoOpExpressionReferenceTransform.INSTANCE.equals(currentTransformer) &&
outerPlanTransformers.size() == 1) {
+ ILogicalExpressionReferenceTransformWithCondition transformer
= outerPlanTransformers.iterator().next();
+ if (transformer instanceof LogicalExpressionReferenceTransform
t && op.acceptExpressionTransform(ContainsMetaFunctionVisitor.INSTANCE)) {
+ Set<LogicalVariable> liveVars = new HashSet<>();
+ VariableUtilities.getLiveVariables(op, liveVars);
+ if (liveVars.contains(t.getMetaVar())) {
+ rewritten |= op.acceptExpressionTransform(t);
+ return currentTransformer;
+ }
}
}
}
@@ -208,6 +233,10 @@
this.metaVar = metaVar;
}
+ public LogicalVariable getMetaVar() {
+ return metaVar;
+ }
+
@Override
public void setVariableRequired() {
this.variableRequired = true;
@@ -393,3 +422,15 @@
return false;
}
}
+
+class ContainsMetaFunctionVisitor implements
ILogicalExpressionReferenceTransform {
+
+ static final ContainsMetaFunctionVisitor INSTANCE = new
ContainsMetaFunctionVisitor();
+
+ @Override
+ public boolean transform(Mutable<ILogicalExpression> expression) throws
AlgebricksException {
+ ILogicalExpression expr = expression.get();
+ return (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL)
+ && ((AbstractFunctionCallExpression)
expr).getFunctionIdentifier().equals(BuiltinFunctions.META);
+ }
+}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
index fcd4325..89301ac 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
@@ -163,7 +163,7 @@
// Standard execution flags.
private boolean executeQuery;
- private final boolean generateJobSpec;
+ private boolean generateJobSpec;
private final boolean optimize;
private long maxWarnings;
private long timeout;
@@ -311,6 +311,10 @@
return generateJobSpec;
}
+ public void setGenerateJobSpec(boolean generateJobSpec) {
+ this.generateJobSpec = generateJobSpec;
+ }
+
/**
* Specify all out-of-band settings at once. For convenience of older code.
*/
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 1b1364b..53d2ec4 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -3450,9 +3450,12 @@
"Transform function definition can not use
collections/views");
}
validateTransformFunction(metadataProvider,
rewrittenQuery, sourceLoc);
+ boolean generateJobSpec =
sessionOutput.config().isGenerateJobSpec();
+ sessionOutput.config().setGenerateJobSpec(false);
apiFramework.compileQuery(hcc, metadataProvider, (Query)
rewrittenQuery.first,
rewrittenQuery.second, null, sessionOutput, null,
null, responsePrinter, warningCollector,
requestParameters, jobFlags);
+ sessionOutput.config().setGenerateJobSpec(generateJobSpec);
}
appCtx.getReceptionist().ensureAuthorized(requestParameters,
metadataProvider);
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/meta/meta_in_subplan_1.sqlpp
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/meta/meta_in_subplan_1.sqlpp
new file mode 100644
index 0000000..f8cdd0a
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/meta/meta_in_subplan_1.sqlpp
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: tests that meta() inside a subplan can be resolved to the meta
of the dataset in the outer plan.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MetaType as open {
+ id:int32
+};
+
+create type DataType as open {
+ id:int32,
+ text: string
+};
+
+create dataset ds(DataType) with meta(MetaType) primary key id;
+
+CREATE OR REPLACE TRANSFORM FUNCTION trans_func(input) {
+ SELECT VALUE d FROM (
+ SELECT * FROM [input] x
+ WHERE meta().id LIKE "id%"
+ ) as d
+ LIMIT 1
+};
+
+select trans_func(ds) from ds;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/results/meta/meta_in_subplan_1.plan
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/meta/meta_in_subplan_1.plan
new file mode 100644
index 0000000..38cfc1a
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/results/meta/meta_in_subplan_1.plan
@@ -0,0 +1,35 @@
+distribute result [$$55]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$55] <- [{"$1": $$54}] project: [$$55]
+ -- ASSIGN |PARTITIONED|
+ project ([$$54])
+ -- STREAM_PROJECT |PARTITIONED|
+ subplan {
+ aggregate [$$54] <- [listify($$52)]
+ -- AGGREGATE |LOCAL|
+ limit 1
+ -- STREAM_LIMIT |LOCAL|
+ assign [$$52] <- [{"x": $$x}] project: [$$52]
+ -- ASSIGN |LOCAL|
+ unnest $$x <-
scan-collection(ordered-list-constructor($$ds)) project: [$$x]
+ -- UNNEST |LOCAL|
+ select (and(lt($$56, "ie"), ge($$56, "id")))
project: [$$ds]
+ -- STREAM_SELECT |LOCAL|
+ nested tuple source
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SUBPLAN |PARTITIONED|
+ assign [$$56] <- [$$58.getField(0)] project: [$$ds, $$56]
+ -- ASSIGN |PARTITIONED|
+ project ([$$ds, $$58])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$57, $$ds, $$58] <- test.ds
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.4.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.4.query.sqlpp
index bb016f7..038a29a 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.4.query.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/meta/meta_after_gby/meta_after_gby.4.query.sqlpp
@@ -26,5 +26,5 @@
FROM DS AS d
GROUP BY d.hobby AS hobby
-SELECT hobby, COUNT(meta().id)
+SELECT hobby, COUNT(meta().id) AS count
ORDER BY hobby;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/meta/meta_after_gby/meta_after_gby.4.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/meta/meta_after_gby/meta_after_gby.4.adm
new file mode 100644
index 0000000..bf1932c
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/meta/meta_after_gby/meta_after_gby.4.adm
@@ -0,0 +1,6 @@
+{ "hobby": "basketball", "count": 1 }
+{ "hobby": "bowling", "count": 1 }
+{ "hobby": "hiking", "count": 2 }
+{ "hobby": "reading", "count": 2 }
+{ "hobby": "soccer", "count": 1 }
+{ "hobby": "tennis", "count": 2 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index c002543..826c62a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -14635,7 +14635,6 @@
<test-case FilePath="meta">
<compilation-unit name="meta_after_gby">
<output-dir compare="Text">meta_after_gby</output-dir>
- <expected-error>Compilation error: No source collection found for
META(): collection not supported or cannot reference collection (in line 29, at
column 21)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="meta">
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21163?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: lumina
Gerrit-Change-Id: I25b7972b6f4a7602cdecdfa6f055bf7ad41b51c7
Gerrit-Change-Number: 21163
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <[email protected]>
