>From Peeyush Gupta <[email protected]>: Peeyush Gupta has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19728 )
Change subject: [ASTERIXDB-3603] Metadata and language changes for transform functions ...................................................................... [ASTERIXDB-3603] Metadata and language changes for transform functions - user model changes: yes - storage format changes: no - interface changes: yes Ext-ref: MB-63039 Change-Id: Ifa0f1d4f83cb9f94f1a29c8b0b0943a1f133e7d1 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19728 Tested-by: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/CloneAndSubstituteVariablesVisitor.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDecl.java M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java M asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java 10 files changed, 81 insertions(+), 13 deletions(-) Approvals: Ali Alsuliman: Looks good to me, approved Jenkins: Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 326191c..dcfd810 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -3340,7 +3340,7 @@ function = new Function(functionSignature, paramNames, paramTypes, returnTypeSignature, null, FunctionKind.SCALAR.toString(), library.getLanguage(), libraryDatabaseName, libraryDataverseName, libraryName, externalIdentifier, cfs.getNullCall(), - cfs.getDeterministic(), cfs.getResources(), dependencies, creator); + cfs.getDeterministic(), cfs.getResources(), dependencies, creator, false); } else { List<Pair<VarIdentifier, TypeExpression>> paramList = cfs.getParameters(); int paramCount = paramList.size(); @@ -3359,7 +3359,8 @@ // Check whether the function is usable: // create a function declaration for this function, // and a query body calls this function with each argument set to 'missing' - FunctionDecl fd = new FunctionDecl(functionSignature, paramVars, cfs.getFunctionBodyExpression(), true); + FunctionDecl fd = new FunctionDecl(functionSignature, paramVars, cfs.getFunctionBodyExpression(), true, + cfs.isTransform()); fd.setSourceLocation(sourceLoc); Query wrappedQuery = queryRewriter.createFunctionAccessorQuery(fd); @@ -3379,7 +3380,7 @@ newInlineTypes = Collections.emptyMap(); function = new Function(functionSignature, paramNames, null, null, cfs.getFunctionBody(), FunctionKind.SCALAR.toString(), compilationProvider.getParserFactory().getLanguage(), null, - null, null, null, null, null, null, dependencies, creator); + null, null, null, null, null, null, dependencies, creator, cfs.isTransform()); } if (existingFunction == null) { diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java index e0623a1..74b7bbd 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java @@ -63,9 +63,11 @@ private final boolean replaceIfExists; private final boolean ifNotExists; + private final boolean transform; public CreateFunctionStatement(FunctionSignature signature, List<Pair<VarIdentifier, TypeExpression>> paramList, - String functionBody, Expression functionBodyExpression, boolean replaceIfExists, boolean ifNotExists) { + String functionBody, Expression functionBodyExpression, boolean replaceIfExists, boolean ifNotExists, + boolean transform) { this.signature = signature; this.functionBody = functionBody; this.functionBodyExpression = functionBodyExpression; @@ -77,6 +79,7 @@ this.options = null; this.replaceIfExists = replaceIfExists; this.ifNotExists = ifNotExists; + this.transform = transform; } public CreateFunctionStatement(FunctionSignature signature, List<Pair<VarIdentifier, TypeExpression>> paramList, @@ -93,6 +96,7 @@ this.functionBodyExpression = null; this.replaceIfExists = replaceIfExists; this.ifNotExists = ifNotExists; + this.transform = false; } public boolean getReplaceIfExists() { @@ -196,6 +200,10 @@ return Category.DDL; } + public boolean isTransform() { + return transform; + } + private IAdmNode getOption(String optionName) { return options != null ? options.get(optionName) : null; } diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDecl.java index 2ef11ad..1a924ee 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDecl.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDecl.java @@ -34,13 +34,20 @@ private Expression funcBody; private Expression funcBodyNormalized; private final boolean isStored; + private final boolean transform; public FunctionDecl(FunctionSignature signature, List<VarIdentifier> paramList, Expression funcBody, - boolean isStored) { + boolean isStored, boolean transform) { this.signature = signature; this.paramList = paramList; this.funcBody = funcBody; this.isStored = isStored; + this.transform = transform; + } + + public FunctionDecl(FunctionSignature signature, List<VarIdentifier> paramList, Expression funcBody, + boolean isStored) { + this(signature, paramList, funcBody, isStored, false); } public FunctionSignature getSignature() { @@ -72,6 +79,10 @@ return isStored; } + public boolean isTransform() { + return transform; + } + @Override public int hashCode() { return signature.hashCode(); diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/CloneAndSubstituteVariablesVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/CloneAndSubstituteVariablesVisitor.java index 06f22b7..4f09722 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/CloneAndSubstituteVariablesVisitor.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/CloneAndSubstituteVariablesVisitor.java @@ -179,7 +179,8 @@ } Pair<ILangExpression, VariableSubstitutionEnvironment> p1 = fd.getFuncBody().accept(this, env); - FunctionDecl newF = new FunctionDecl(fd.getSignature(), newList, (Expression) p1.first, fd.isStored()); + FunctionDecl newF = + new FunctionDecl(fd.getSignature(), newList, (Expression) p1.first, fd.isStored(), fd.isTransform()); newF.setSourceLocation(fd.getSourceLocation()); return new Pair<>(newF, env); } diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java index 1d43d0b..9941dde 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/DeepCopyVisitor.java @@ -267,7 +267,7 @@ @Override public FunctionDecl visit(FunctionDecl fd, Void arg) throws CompilationException { FunctionDecl copy = new FunctionDecl(fd.getSignature(), fd.getParamList(), - (Expression) fd.getFuncBody().accept(this, arg), fd.isStored()); + (Expression) fd.getFuncBody().accept(this, arg), fd.isStored(), fd.isTransform()); copy.setSourceLocation(fd.getSourceLocation()); return copy; } diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj index a91b3d5..8afe035 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj +++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj @@ -1868,13 +1868,18 @@ CreateFunctionStatement stmt = null; } { - <FUNCTION> stmt = FunctionSpecification(startStmtToken, orReplace) + <TRANSFORM> <FUNCTION> stmt = FunctionSpecification(startStmtToken, orReplace, true) + { + return stmt; + } + | + <FUNCTION> stmt = FunctionSpecification(startStmtToken, orReplace, false) { return stmt; } } -CreateFunctionStatement FunctionSpecification(Token startStmtToken, boolean orReplace) throws ParseException: +CreateFunctionStatement FunctionSpecification(Token startStmtToken, boolean orReplace, boolean transform) throws ParseException: { FunctionSignature signature = null; FunctionName fctName = null; @@ -1929,7 +1934,7 @@ getCurrentScope().addFunctionDescriptor(signature, false); removeCurrentScope(); ensureNoTypeDeclsInFunction(fctName.function, params, returnType, startStmtToken); - stmt = new CreateFunctionStatement(signature, params, functionBody, functionBodyExpr, orReplace, ifNotExists); + stmt = new CreateFunctionStatement(signature, params, functionBody, functionBodyExpr, orReplace, ifNotExists, transform); } ) | @@ -6106,6 +6111,7 @@ | <WITH : "with"> | <WRITE : "write"> | <COPY : "copy"> + | <TRANSFORM : "transform"> } <DEFAULT,IN_DBL_BRACE> diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java index 70bf83a..8db32d6 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java @@ -203,7 +203,7 @@ public void dropFunction(FunctionSignature signature) { Function function = new Function(signature, null, null, null, null, null, null, null, null, null, null, false, - false, null, null, null); + false, null, null, null, false); droppedCache.addFunctionIfNotExists(function); logAndApply(new MetadataLogicalOperation(function, false)); } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java index 338e00d..31a7f9b 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java @@ -53,6 +53,7 @@ public static final String FIELD_NAME_DEFAULT = "Default"; public static final String FIELD_NAME_DEFINITION = "Definition"; public static final String FIELD_NAME_DEPENDENCIES = "Dependencies"; + public static final String FIELD_NAME_IS_TRANSFORM = "IsTransform"; public static final String FIELD_NAME_DERIVED = "Derived"; public static final String FIELD_NAME_DESCRIPTION = "Description"; public static final String FIELD_NAME_EXTERNAL_DETAILS = "ExternalDetails"; diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java index 3c1515b..c19b542 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java @@ -51,12 +51,13 @@ private final Map<String, String> resources; private final List<List<DependencyFullyQualifiedName>> dependencies; private final Creator creator; + private final boolean transform; public Function(FunctionSignature signature, List<String> paramNames, List<TypeSignature> paramTypes, TypeSignature returnType, String functionBody, String functionKind, String language, String libraryDatabaseName, DataverseName libraryDataverseName, String libraryName, List<String> externalIdentifier, Boolean nullCall, Boolean deterministic, Map<String, String> resources, - List<List<DependencyFullyQualifiedName>> dependencies, Creator creator) { + List<List<DependencyFullyQualifiedName>> dependencies, Creator creator, boolean transform) { this.signature = signature; this.paramNames = paramNames; this.paramTypes = paramTypes; @@ -75,6 +76,7 @@ ? Arrays.asList(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()) : dependencies; this.creator = creator; + this.transform = transform; } public FunctionSignature getSignature() { @@ -168,6 +170,10 @@ return creator; } + public boolean isTransform() { + return transform; + } + @Override public Function addToCache(MetadataCache cache) { return cache.addFunctionIfNotExists(this); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java index 2741d12..2319cb0 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java @@ -206,9 +206,14 @@ FunctionSignature signature = new FunctionSignature(databaseName, dataverseName, functionName, arity); Creator creator = Creator.createOrDefault(functionRecord); + int isTransformIndex = functionRecord.getType().getFieldIndex(MetadataRecordTypes.FIELD_NAME_IS_TRANSFORM); + boolean transform = false; + if (isTransformIndex >= 0) { + transform = ((ABoolean) functionRecord.getValueByPos(isTransformIndex)).getBoolean(); + } return new Function(signature, paramNames, paramTypes, returnType, definition, functionKind, language, libraryDatabaseName, libraryDataverseName, libraryName, externalIdentifier, nullCall, deterministic, - resources, dependencies, creator); + resources, dependencies, creator, transform); } private List<TypeSignature> getParamTypes(ARecord functionRecord, String functionDatabaseName, @@ -435,6 +440,7 @@ writeNullCall(function); writeDeterministic(function); writeFunctionCreator(function); + writeIsTransform(function); } protected void writeResources(Function function) throws HyracksDataException { @@ -722,4 +728,15 @@ recordBuilder.addField(fieldName, fieldValue); } } + + private void writeIsTransform(Function function) throws HyracksDataException { + if (function.isTransform()) { + fieldName.reset(); + aString.setValue(MetadataRecordTypes.FIELD_NAME_IS_TRANSFORM); + stringSerde.serialize(aString, fieldName.getDataOutput()); + fieldValue.reset(); + booleanSerde.serialize(ABoolean.TRUE, fieldValue.getDataOutput()); + recordBuilder.addField(fieldName, fieldValue); + } + } } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19728 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: Ifa0f1d4f83cb9f94f1a29c8b0b0943a1f133e7d1 Gerrit-Change-Number: 19728 Gerrit-PatchSet: 6 Gerrit-Owner: Peeyush Gupta <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Peeyush Gupta <[email protected]> Gerrit-MessageType: merged
