Yingyi Bu has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/1008
Change subject: ASTERIXDB-1536: supports fully qualified dataset path in SQL++. ...................................................................... ASTERIXDB-1536: supports fully qualified dataset path in SQL++. Change-Id: I93c7187b3a363a82dbfa225eb67ab526e04aa2dd --- M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java M asterixdb/asterix-app/src/test/resources/parserts/queries_sqlpp/columnalias2.sqlpp M asterixdb/asterix-app/src/test/resources/parserts/results_parser_sqlpp/columnalias2.ast A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.2.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.3.query.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java 8 files changed, 193 insertions(+), 37 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/08/1008/1 diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java index 4965197..295eb41 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java @@ -93,13 +93,15 @@ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr; for (Mutable<ILogicalExpression> funcArgRef : funcExpr.getArguments()) { if (rewriteExpressionReference(op, funcArgRef, context)) { - context.computeAndSetTypeEnvironmentForOperator(op); changed = true; } } // Cleans up extra scan-collections if there is. - cleanupScanCollectionForDataset(funcExpr); + if (changed) { + cleanupScanCollectionForDataset(funcExpr); + context.computeAndSetTypeEnvironmentForOperator(op); + } // Does the actual resolution. return changed || resolve(op, context, exprRef); @@ -146,8 +148,11 @@ throw new AlgebricksException( "Undefined alias (variable) reference for identifier " + unresolvedVarName); } - // Rewrites the "resolve" function to a "dataset" function. + // Rewrites the "resolve" function to a "dataset" function and only keep the dataset name argument. func.setFunctionInfo(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.DATASET)); + Mutable<ILogicalExpression> datasetNameExpression = func.getArguments().get(0); + func.getArguments().clear(); + func.getArguments().add(datasetNameExpression); } else { // Rewrites to field-access-by-names. Pair<LogicalVariable, List<String>> varAndPath = varAccessCandidates.iterator().next(); diff --git a/asterixdb/asterix-app/src/test/resources/parserts/queries_sqlpp/columnalias2.sqlpp b/asterixdb/asterix-app/src/test/resources/parserts/queries_sqlpp/columnalias2.sqlpp index 8eeea85..f74cf47 100644 --- a/asterixdb/asterix-app/src/test/resources/parserts/queries_sqlpp/columnalias2.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/parserts/queries_sqlpp/columnalias2.sqlpp @@ -21,4 +21,4 @@ GROUP BY root.id WITH u AS root.time HAVING root.orders > 0 -ORDER BY u; \ No newline at end of file +ORDER BY u; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/parserts/results_parser_sqlpp/columnalias2.ast b/asterixdb/asterix-app/src/test/resources/parserts/results_parser_sqlpp/columnalias2.ast index 0b7ccd2..89e3b4b 100644 --- a/asterixdb/asterix-app/src/test/resources/parserts/results_parser_sqlpp/columnalias2.ast +++ b/asterixdb/asterix-app/src/test/resources/parserts/results_parser_sqlpp/columnalias2.ast @@ -2,18 +2,12 @@ SELECT [ FunctionCall null.SQRT@1[ OperatorExpr [ - FieldAccessor [ - FunctionCall Metadata.dataset@1[ - LiteralExpr [STRING] [t] - ] - Field=a + FunctionCall Metadata.dataset@1[ + LiteralExpr [STRING] [t.a] ] * - FieldAccessor [ - FunctionCall Metadata.dataset@1[ - LiteralExpr [STRING] [t] - ] - Field=b + FunctionCall Metadata.dataset@1[ + LiteralExpr [STRING] [t.b] ] ] ] diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.1.ddl.sqlpp new file mode 100644 index 0000000..193d6b4 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.1.ddl.sqlpp @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop database test if exists; +create database test; + +use test; + + +create type test.AddressType as +{ + number : int64, + street : string, + city : string +} + +create type test.CustomerType as + closed { + cid : int64, + name : string, + cashBack : int64, + age : int64?, + address : AddressType?, + lastorder : { + oid : int64, + total : float + } + +} + +create type test.OrderType as +{ + oid : int64, + cid : int64, + orderstatus : string, + orderpriority : string, + clerk : string, + total : float, + items : [int64] +} + +create external table Customers(CustomerType) using localfs((`path`=`asterix_nc1://data/nontagged/customerData.json`),(`format`=`adm`)); + +create external table Orders(OrderType) using localfs((`path`=`asterix_nc1://data/nontagged/orderData.json`),(`format`=`adm`)); + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.2.update.sqlpp new file mode 100644 index 0000000..6c98c1e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.2.update.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.3.query.sqlpp new file mode 100644 index 0000000..c33377e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/custord/join_q_08/join_q_08.3.query.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** This query is to test fully qualified path for datasets. */ + +SELECT c.name AS cust_name, + c.age AS cust_age, + o.total AS order_total, + [o.oid,o.cid] AS orderList +FROM test.Customers c JOIN test.Orders o ON c.cid = o.cid +ORDER BY c.name,o.total +; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index e3d3102..6a19c93 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -1349,7 +1349,12 @@ <test-case FilePath="custord"> <compilation-unit name="join_q_07"> <output-dir compare="Text">join_q_06</output-dir> - <expected-error>Undefined alias (variable) reference for identifier c</expected-error> + <expected-error>Cannot find dataset c in dataverse test nor a variable with name c</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="custord"> + <compilation-unit name="join_q_08"> + <output-dir compare="Text">join_q_01</output-dir> </compilation-unit> </test-case> <test-case FilePath="custord"> @@ -2855,7 +2860,7 @@ <output-dir compare="Text">partition-by-nonexistent-field</output-dir> <expected-error>java.lang.NullPointerException</expected-error> <expected-error>Cannot find dataset</expected-error> - <expected-error>Undefined alias (variable) reference for identifier testds</expected-error> + <expected-error>Cannot find dataset testds in dataverse test nor a variable with name testds</expected-error> </compilation-unit> </test-case> <test-case FilePath="misc"> @@ -6666,7 +6671,7 @@ <test-case FilePath="user-defined-functions"> <compilation-unit name="udf30"> <output-dir compare="Text">udf30</output-dir> - <expected-error>Undefined alias (variable) reference for identifier y</expected-error> + <expected-error>Cannot find dataset y in dataverse null nor a variable with name y!</expected-error> </compilation-unit> </test-case> <test-case FilePath="user-defined-functions"> diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java index 7082c25..ba26ea4 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java @@ -26,8 +26,10 @@ import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.lang.common.base.Expression; +import org.apache.asterix.lang.common.base.Expression.Kind; import org.apache.asterix.lang.common.base.ILangExpression; import org.apache.asterix.lang.common.expression.CallExpr; +import org.apache.asterix.lang.common.expression.FieldAccessor; import org.apache.asterix.lang.common.expression.LiteralExpr; import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.literal.StringLiteral; @@ -41,8 +43,8 @@ public class VariableCheckAndRewriteVisitor extends AbstractSqlppExpressionScopingVisitor { - protected final FunctionSignature datasetFunction = new FunctionSignature(MetadataConstants.METADATA_DATAVERSE_NAME, - "dataset", 1); + protected final FunctionSignature datasetFunction = + new FunctionSignature(MetadataConstants.METADATA_DATAVERSE_NAME, "dataset", 1); protected final boolean overwrite; protected final AqlMetadataProvider metadataProvider; @@ -62,23 +64,65 @@ } @Override + public Expression visit(FieldAccessor fa, ILangExpression arg) throws AsterixException { + Expression leadingExpr = fa.getExpr(); + if (leadingExpr.getKind() != Kind.VARIABLE_EXPRESSION) { + fa.setExpr(leadingExpr.accept(this, fa)); + return fa; + } else { + VariableExpr varExpr = (VariableExpr) leadingExpr; + String lastIdentifier = fa.getIdent().getValue(); + Expression resolvedExpr = resolve(varExpr, + /** Resolves within the dataverse that has the same name as the variable name. */ + SqlppVariableUtil.toUserDefinedVariableName(varExpr.getVar().getValue()).getValue(), lastIdentifier, + arg); + if (resolvedExpr.getKind() == Kind.CALL_EXPRESSION) { + CallExpr callExpr = (CallExpr) resolvedExpr; + if (callExpr.getFunctionSignature().equals(datasetFunction)) { + // The field access is resolved to be a dataset access in the form of "dataverse.dataset". + return resolvedExpr; + } + } + fa.setExpr(resolvedExpr); + return fa; + } + } + + @Override public Expression visit(VariableExpr varExpr, ILangExpression arg) throws AsterixException { + return resolve(varExpr, null /** Resolves within the default dataverse. */ + , SqlppVariableUtil.toUserDefinedVariableName(varExpr.getVar().getValue()).getValue(), arg); + } + + // Resolve a variable expression with dataverse name and dataset name. + private Expression resolve(VariableExpr varExpr, String dataverseName, String datasetName, ILangExpression arg) + throws AsterixException { String varName = varExpr.getVar().getValue(); + checkError(varName); + if (!rewriteNeeded(varExpr)) { + return varExpr; + } + Set<VariableExpr> liveVars = SqlppVariableUtil.getLiveUserDefinedVariables(scopeChecker.getCurrentScope()); + boolean resolveAsDataset = resolveDatasetFirst(arg) && datasetExists(dataverseName, datasetName); + if (resolveAsDataset) { + return wrapWithDatasetFunction(dataverseName, datasetName); + } else if (liveVars.isEmpty()) { + // If no available dataset nor in-scope variable to resolve to, we throw an error. + throw new AsterixException("Cannot find dataset " + datasetName + " in dataverse " + + (dataverseName == null ? metadataProvider.getDefaultDataverseName() : dataverseName) + + " nor a variable with name " + datasetName + "!"); + } + return wrapWithResolveFunction(varExpr, liveVars); + } + + // Checks whether we need to error the variable reference, e.g., the variable is referred + // in a LIMIT clause. + private void checkError(String varName) throws AsterixException { if (scopeChecker.isInForbiddenScopes(varName)) { throw new AsterixException( "Inside limit clauses, it is disallowed to reference a variable having the same name" + " as any variable bound in the same scope as the limit clause."); } - if (!rewriteNeeded(varExpr)) { - return varExpr; - } - boolean resolveAsDataset = resolveDatasetFirst(arg) - && datasetExists(SqlppVariableUtil.toUserDefinedVariableName(varName).getValue()); - if (resolveAsDataset) { - return wrapWithDatasetFunction(varExpr); - } - Set<VariableExpr> liveVars = SqlppVariableUtil.getLiveUserDefinedVariables(scopeChecker.getCurrentScope()); - return wrapWithResolveFunction(varExpr, liveVars); } // For From/Join/UNNEST/NEST, we resolve the undefined identifier reference as dataset reference first. @@ -101,26 +145,25 @@ } } - private Expression wrapWithDatasetFunction(VariableExpr expr) throws AsterixException { + private Expression wrapWithDatasetFunction(String dataverseName, String datasetName) throws AsterixException { + String fullyQualifiedName = dataverseName == null ? datasetName : dataverseName + "." + datasetName; List<Expression> argList = new ArrayList<>(); - //Ignore the parser-generated prefix "$" for a dataset. - String varName = SqlppVariableUtil.toUserDefinedVariableName(expr.getVar()).getValue(); - argList.add(new LiteralExpr(new StringLiteral(varName))); + argList.add(new LiteralExpr(new StringLiteral(fullyQualifiedName))); return new CallExpr(datasetFunction, argList); } - private boolean datasetExists(String name) throws AsterixException { + private boolean datasetExists(String dataverseName, String datasetName) throws AsterixException { try { - if (metadataProvider.findDataset(null, name) != null) { + if (metadataProvider.findDataset(dataverseName, datasetName) != null) { return true; } - return pathDatasetExists(name); + return fullyQualifiedDatasetNameExists(datasetName); } catch (AlgebricksException e) { throw new AsterixException(e); } } - private boolean pathDatasetExists(String name) throws AlgebricksException { + private boolean fullyQualifiedDatasetNameExists(String name) throws AlgebricksException { if (!name.contains(".")) { return false; } -- To view, visit https://asterix-gerrit.ics.uci.edu/1008 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I93c7187b3a363a82dbfa225eb67ab526e04aa2dd Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Yingyi Bu <buyin...@gmail.com>