Repository: systemml Updated Branches: refs/heads/master e2875cae4 -> c250adab3
[SYSTEMML-2315] JMLC API extensions for passing multiple dml scripts This patch extends the JMLC API for multiple dml scripts. Specifically, we now allow scripts with import (i.e., source) statements where all additional scripts are handed over in memory to avoid invalid relative file lookups (because most JMLC applications load there scripts from resource streams). Furthermore, this includes a major refactoring of the handling of dml/pydml imports to avoid unnecessary redundancy. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/c250adab Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/c250adab Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/c250adab Branch: refs/heads/master Commit: c250adab3ee17aa767fbe8af966ec8cabe9ba797 Parents: e2875ca Author: Matthias Boehm <mboe...@gmail.com> Authored: Fri May 11 15:54:07 2018 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Fri May 11 15:54:07 2018 -0700 ---------------------------------------------------------------------- .../org/apache/sysml/api/jmlc/Connection.java | 22 ++++- .../org/apache/sysml/parser/ParserFactory.java | 18 ++-- .../parser/common/CommonSyntacticValidator.java | 64 +++++++++++-- .../sysml/parser/common/StatementInfo.java | 2 - .../sysml/parser/dml/DmlSyntacticValidator.java | 64 +++---------- .../parser/pydml/PydmlSyntacticValidator.java | 60 +++--------- .../sysml/runtime/util/UtilFunctions.java | 6 +- .../functions/jmlc/NamespaceFunctionTest.java | 99 ++++++++++++++++++++ src/test/scripts/functions/jmlc/bar1.dml | 27 ++++++ src/test/scripts/functions/jmlc/bar2.dml | 29 ++++++ src/test/scripts/functions/jmlc/foo.dml | 30 ++++++ .../functions/jmlc/ZPackageSuite.java | 1 + 12 files changed, 302 insertions(+), 120 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/main/java/org/apache/sysml/api/jmlc/Connection.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java index 8684e72..7eb262f 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java +++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java @@ -26,7 +26,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Arrays; -import java.util.HashMap; +import java.util.Collections; import java.util.Map; import org.apache.hadoop.fs.FileSystem; @@ -201,7 +201,7 @@ public class Connection implements Closeable * @return PreparedScript object representing the precompiled script */ public PreparedScript prepareScript( String script, String[] inputs, String[] outputs, boolean parsePyDML) { - return prepareScript(script, new HashMap<String,String>(), inputs, outputs, parsePyDML); + return prepareScript(script, Collections.emptyMap(), inputs, outputs, parsePyDML); } /** @@ -215,6 +215,21 @@ public class Connection implements Closeable * @return PreparedScript object representing the precompiled script */ public PreparedScript prepareScript( String script, Map<String, String> args, String[] inputs, String[] outputs, boolean parsePyDML) { + return prepareScript(script, Collections.emptyMap(), args, inputs, outputs, parsePyDML); + } + + /** + * Prepares (precompiles) a script, sets input parameter values, and registers input and output variables. + * + * @param script string representing of the DML or PyDML script + * @param nsscript map (name, script) of the DML or PyDML namespace scripts + * @param args map of input parameters ($) and their values + * @param inputs string array of input variables to register + * @param outputs string array of output variables to register + * @param parsePyDML {@code true} if PyDML, {@code false} if DML + * @return PreparedScript object representing the precompiled script + */ + public PreparedScript prepareScript(String script, Map<String,String> nsscripts, Map<String, String> args, String[] inputs, String[] outputs, boolean parsePyDML) { DMLScript.SCRIPT_TYPE = parsePyDML ? ScriptType.PYDML : ScriptType.DML; //check for valid names of passed arguments @@ -235,7 +250,8 @@ public class Connection implements Closeable Program rtprog = null; try { //parsing - ParserWrapper parser = ParserFactory.createParser(parsePyDML ? ScriptType.PYDML : ScriptType.DML); + ParserWrapper parser = ParserFactory.createParser( + parsePyDML ? ScriptType.PYDML : ScriptType.DML, nsscripts); DMLProgram prog = parser.parse(null, script, args); //language validate http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/main/java/org/apache/sysml/parser/ParserFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/ParserFactory.java b/src/main/java/org/apache/sysml/parser/ParserFactory.java index 2d4e273..6b2b686 100644 --- a/src/main/java/org/apache/sysml/parser/ParserFactory.java +++ b/src/main/java/org/apache/sysml/parser/ParserFactory.java @@ -19,6 +19,9 @@ package org.apache.sysml.parser; +import java.util.Collections; +import java.util.Map; + import org.apache.sysml.api.mlcontext.ScriptType; import org.apache.sysml.parser.common.CommonSyntacticValidator; import org.apache.sysml.parser.dml.DMLParserWrapper; @@ -26,24 +29,25 @@ import org.apache.sysml.parser.pydml.PyDMLParserWrapper; public class ParserFactory { + public static ParserWrapper createParser(ScriptType scriptType) { + return createParser(scriptType, Collections.emptyMap()); + } + /** * Factory method for creating parser wrappers * - * @param scriptType - * type of script + * @param scriptType type of script + * @param nsscripts map of namespace scripts (name, script) * @return parser wrapper (DMLParserWrapper or PyDMLParserWrapper) */ - public static ParserWrapper createParser(ScriptType scriptType) { + public static ParserWrapper createParser(ScriptType scriptType, Map<String, String> nsscripts) { ParserWrapper ret = null; - // create the parser instance switch (scriptType) { case DML: ret = new DMLParserWrapper(); break; case PYDML: ret = new PyDMLParserWrapper(); break; } - CommonSyntacticValidator.init(); - + CommonSyntacticValidator.init(nsscripts); return ret; } - } http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java index 3226895..ab9c330 100644 --- a/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java @@ -19,11 +19,13 @@ package org.apache.sysml.parser.common; +import java.io.File; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.regex.Pattern; @@ -45,6 +47,7 @@ import org.apache.sysml.parser.Expression.DataOp; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.parser.FunctionCallIdentifier; +import org.apache.sysml.parser.ImportStatement; import org.apache.sysml.parser.IntIdentifier; import org.apache.sysml.parser.LanguageException; import org.apache.sysml.parser.MultiAssignmentStatement; @@ -63,24 +66,26 @@ import org.apache.sysml.parser.BuiltinConstant; * Contains fields and (helper) methods common to {@link DmlSyntacticValidator} and {@link PydmlSyntacticValidator} */ public abstract class CommonSyntacticValidator { - + private static final String DEF_WORK_DIR = "."; + + //externally loaded dml scripts filename (unmodified) / script + protected static ThreadLocal<HashMap<String, String>> _tScripts = new ThreadLocal<HashMap<String, String>>() { + @Override protected HashMap<String, String> initialValue() { return new HashMap<>(); } + }; + //imported scripts to prevent infinite recursion, modified filename / namespace + protected static ThreadLocal<HashMap<String, String>> _f2NS = new ThreadLocal<HashMap<String, String>>() { + @Override protected HashMap<String, String> initialValue() { return new HashMap<>(); } + }; + protected final CustomErrorListener errorListener; protected final String currentFile; - protected String _workingDir = "."; //current working directory + protected String _workingDir = DEF_WORK_DIR; protected Map<String,String> argVals = null; protected String sourceNamespace = null; - // Track imported scripts to prevent infinite recursion - protected static ThreadLocal<HashMap<String, String>> _scripts = new ThreadLocal<HashMap<String, String>>() { - @Override protected HashMap<String, String> initialValue() { return new HashMap<>(); } - }; // Map namespaces to full paths as defined only from source statements in this script (i.e., currentFile) protected HashMap<String, String> sources; // Names of new internal and external functions defined in this script (i.e., currentFile) protected Set<String> functions; - - public static void init() { - _scripts.get().clear(); - } public CommonSyntacticValidator(CustomErrorListener errorListener, Map<String,String> argVals, String sourceNamespace, Set<String> prepFunctions) { this.errorListener = errorListener; @@ -90,6 +95,17 @@ public abstract class CommonSyntacticValidator { sources = new HashMap<>(); functions = (null != prepFunctions) ? prepFunctions : new HashSet<>(); } + + public static void init() { + _f2NS.get().clear(); + } + + public static void init(Map<String, String> scripts) { + _f2NS.get().clear(); + _tScripts.get().clear(); + for( Entry<String,String> e : scripts.entrySet() ) + _tScripts.get().put(getDefWorkingFilePath(e.getKey()), e.getValue()); + } protected void notifyErrorListeners(String message, Token op) { if (!DMLScript.VALIDATOR_IGNORE_ISSUES) { @@ -150,6 +166,24 @@ public abstract class CommonSyntacticValidator { String path = sources.get(namespace); return (path != null && path.length() > 0) ? path : namespace; } + + public String getWorkingFilePath(String filePath) { + return getWorkingFilePath(filePath, _workingDir); + } + + public static String getDefWorkingFilePath(String filePath) { + return getWorkingFilePath(filePath, DEF_WORK_DIR); + } + + private static String getWorkingFilePath(String filePath, String workingDir) { + return !new File(filePath).isAbsolute() ? + workingDir + File.separator + filePath : filePath; + } + + public String getNamespaceSafe(Token ns) { + return (ns != null && ns.getText() != null && !ns.getText().isEmpty()) ? + ns.getText() : DMLProgram.DEFAULT_NAMESPACE; + } protected void validateNamespace(String namespace, String filePath, ParserRuleContext ctx) { if (!sources.containsKey(namespace)) { @@ -161,6 +195,16 @@ public abstract class CommonSyntacticValidator { notifyErrorListeners("Namespace Conflict: '" + namespace + "' already defined as " + sources.get(namespace), ctx.start); } } + + protected void setupContextInfo(StatementInfo info, String namespace, String filePath, String filePath2, DMLProgram prog ) { + info.namespaces = new HashMap<>(); + info.namespaces.put(getQualifiedNamespace(namespace), prog); + ImportStatement istmt = new ImportStatement(); + istmt.setCompletePath(filePath); + istmt.setFilename(filePath2); + istmt.setNamespace(namespace); + info.stmt = istmt; + } protected void setFileLineColumn(Expression expr, ParserRuleContext ctx) { expr.setCtxValuesAndFilename(ctx, currentFile); http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/main/java/org/apache/sysml/parser/common/StatementInfo.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/common/StatementInfo.java b/src/main/java/org/apache/sysml/parser/common/StatementInfo.java index aad5a6a..941298b 100644 --- a/src/main/java/org/apache/sysml/parser/common/StatementInfo.java +++ b/src/main/java/org/apache/sysml/parser/common/StatementInfo.java @@ -34,7 +34,6 @@ import org.apache.sysml.parser.Statement; */ public class StatementInfo { - public Statement stmt = null; // Valid only for import statements @@ -43,5 +42,4 @@ public class StatementInfo { // Valid only for function statement //public String namespace = DMLProgram.DEFAULT_NAMESPACE; public String functionName = ""; - } http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java index ba5b2db..5624f2a 100644 --- a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java @@ -19,7 +19,6 @@ package org.apache.sysml.parser.dml; -import java.io.File; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -44,7 +43,6 @@ import org.apache.sysml.parser.ForStatement; import org.apache.sysml.parser.FunctionCallIdentifier; import org.apache.sysml.parser.FunctionStatement; import org.apache.sysml.parser.IfStatement; -import org.apache.sysml.parser.ImportStatement; import org.apache.sysml.parser.IndexedIdentifier; import org.apache.sysml.parser.IterablePredicate; import org.apache.sysml.parser.LanguageException; @@ -110,6 +108,7 @@ import org.apache.sysml.parser.dml.DmlParser.TypedArgNoAssignContext; import org.apache.sysml.parser.dml.DmlParser.UnaryExpressionContext; import org.apache.sysml.parser.dml.DmlParser.ValueTypeContext; import org.apache.sysml.parser.dml.DmlParser.WhileStatementContext; +import org.apache.sysml.runtime.util.UtilFunctions; public class DmlSyntacticValidator extends CommonSyntacticValidator implements DmlListener { @@ -375,67 +374,39 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D // ----------------------------------------------------------------- - // "src" statment + // "source" statement // ----------------------------------------------------------------- @Override - public void exitImportStatement(ImportStatementContext ctx) - { - //prepare import filepath - String filePath = ctx.filePath.getText(); - String namespace = DMLProgram.DEFAULT_NAMESPACE; - if(ctx.namespace != null && ctx.namespace.getText() != null && !ctx.namespace.getText().isEmpty()) { - namespace = ctx.namespace.getText(); - } - if((filePath.startsWith("\"") && filePath.endsWith("\"")) || - filePath.startsWith("'") && filePath.endsWith("'")) { - filePath = filePath.substring(1, filePath.length()-1); - } - - File file = new File(filePath); - if (!file.isAbsolute()) { - //concatenate working directory to filepath - filePath = _workingDir + File.separator + filePath; - } + public void exitImportStatement(ImportStatementContext ctx) { + String filePath = getWorkingFilePath(UtilFunctions.unquote(ctx.filePath.getText())); + String namespace = getNamespaceSafe(ctx.namespace); validateNamespace(namespace, filePath, ctx); String scriptID = DMLProgram.constructFunctionKey(namespace, filePath); DMLProgram prog = null; - if (!_scripts.get().containsKey(scriptID)) - { - _scripts.get().put(scriptID, namespace); + if (!_f2NS.get().containsKey(scriptID)) { + _f2NS.get().put(scriptID, namespace); try { - prog = (new DMLParserWrapper()).doParse(filePath, null, getQualifiedNamespace(namespace), argVals); - } catch (ParseException e) { + prog = (new DMLParserWrapper()).doParse(filePath, + _tScripts.get().get(filePath), getQualifiedNamespace(namespace), argVals); + } + catch (ParseException e) { notifyErrorListeners(e.getMessage(), ctx.start); return; } - // Custom logic whether to proceed ahead or not. Better than the current exception handling mechanism if(prog == null) { notifyErrorListeners("One or more errors found during importing a program from file " + filePath, ctx.start); return; } - else { - ctx.info.namespaces = new HashMap<>(); - ctx.info.namespaces.put(getQualifiedNamespace(namespace), prog); - ctx.info.stmt = new ImportStatement(); - ((ImportStatement) ctx.info.stmt).setCompletePath(filePath); - ((ImportStatement) ctx.info.stmt).setFilePath(ctx.filePath.getText()); - ((ImportStatement) ctx.info.stmt).setNamespace(namespace); - } + setupContextInfo(ctx.info, namespace, filePath, ctx.filePath.getText(), prog); } - else - { + else { // Skip redundant parsing (to prevent potential infinite recursion) and // create empty program for this context to allow processing to continue. prog = new DMLProgram(); - ctx.info.namespaces = new HashMap<>(); - ctx.info.namespaces.put(getQualifiedNamespace(namespace), prog); - ctx.info.stmt = new ImportStatement(); - ((ImportStatement) ctx.info.stmt).setCompletePath(filePath); - ((ImportStatement) ctx.info.stmt).setFilePath(ctx.filePath.getText()); - ((ImportStatement) ctx.info.stmt).setNamespace(namespace); + setupContextInfo(ctx.info, namespace, filePath, ctx.filePath.getText(), prog); } } @@ -826,12 +797,7 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D @Override public void exitPathStatement(PathStatementContext ctx) { PathStatement stmt = new PathStatement(ctx.pathValue.getText()); - String filePath = ctx.pathValue.getText(); - if((filePath.startsWith("\"") && filePath.endsWith("\"")) || - filePath.startsWith("'") && filePath.endsWith("'")) { - filePath = filePath.substring(1, filePath.length()-1); - } - + String filePath = UtilFunctions.unquote(ctx.pathValue.getText()); _workingDir = filePath; ctx.info.stmt = stmt; } http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java index 117dff1..85adc3a 100644 --- a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java @@ -19,7 +19,6 @@ package org.apache.sysml.parser.pydml; -import java.io.File; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -47,7 +46,6 @@ import org.apache.sysml.parser.ForStatement; import org.apache.sysml.parser.FunctionCallIdentifier; import org.apache.sysml.parser.FunctionStatement; import org.apache.sysml.parser.IfStatement; -import org.apache.sysml.parser.ImportStatement; import org.apache.sysml.parser.IndexedIdentifier; import org.apache.sysml.parser.IntIdentifier; import org.apache.sysml.parser.IterablePredicate; @@ -114,6 +112,7 @@ import org.apache.sysml.parser.pydml.PydmlParser.TypedArgNoAssignContext; import org.apache.sysml.parser.pydml.PydmlParser.UnaryExpressionContext; import org.apache.sysml.parser.pydml.PydmlParser.ValueDataTypeCheckContext; import org.apache.sysml.parser.pydml.PydmlParser.WhileStatementContext; +import org.apache.sysml.runtime.util.UtilFunctions; /** * TODO: Refactor duplicated parser code dml/pydml (entire package). @@ -479,60 +478,34 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements @Override public void exitImportStatement(ImportStatementContext ctx) { - //prepare import filepath - String filePath = ctx.filePath.getText(); - String namespace = DMLProgram.DEFAULT_NAMESPACE; - if(ctx.namespace != null && ctx.namespace.getText() != null && !ctx.namespace.getText().isEmpty()) { - namespace = ctx.namespace.getText(); - } - if((filePath.startsWith("\"") && filePath.endsWith("\"")) || - filePath.startsWith("'") && filePath.endsWith("'")) { - filePath = filePath.substring(1, filePath.length()-1); - } - - File file = new File(filePath); - if (!file.isAbsolute()) { - //concatenate working directory to filepath - filePath = _workingDir + File.separator + filePath; - } + String filePath = getWorkingFilePath(UtilFunctions.unquote(ctx.filePath.getText())); + String namespace = getNamespaceSafe(ctx.namespace); + validateNamespace(namespace, filePath, ctx); String scriptID = DMLProgram.constructFunctionKey(namespace, filePath); DMLProgram prog = null; - if (!_scripts.get().containsKey(scriptID)) - { - _scripts.get().put(scriptID, namespace); + if (!_f2NS.get().containsKey(scriptID)) { + _f2NS.get().put(scriptID, namespace); try { - prog = (new PyDMLParserWrapper()).doParse(filePath, null, getQualifiedNamespace(namespace), argVals); - } catch (ParseException e) { + prog = (new PyDMLParserWrapper()).doParse(filePath, + _tScripts.get().get(filePath), getQualifiedNamespace(namespace), argVals); + } + catch (ParseException e) { notifyErrorListeners(e.getMessage(), ctx.start); return; } - // Custom logic whether to proceed ahead or not. Better than the current exception handling mechanism if(prog == null) { notifyErrorListeners("One or more errors found during importing a program from file " + filePath, ctx.start); return; } - else { - ctx.info.namespaces = new HashMap<>(); - ctx.info.namespaces.put(getQualifiedNamespace(namespace), prog); - ctx.info.stmt = new ImportStatement(); - ((ImportStatement) ctx.info.stmt).setCompletePath(filePath); - ((ImportStatement) ctx.info.stmt).setFilePath(ctx.filePath.getText()); - ((ImportStatement) ctx.info.stmt).setNamespace(namespace); - } + setupContextInfo(ctx.info, namespace, filePath, ctx.filePath.getText(), prog); } - else - { + else { // Skip redundant parsing (to prevent potential infinite recursion) and // create empty program for this context to allow processing to continue. prog = new DMLProgram(); - ctx.info.namespaces = new HashMap<>(); - ctx.info.namespaces.put(getQualifiedNamespace(namespace), prog); - ctx.info.stmt = new ImportStatement(); - ((ImportStatement) ctx.info.stmt).setCompletePath(filePath); - ((ImportStatement) ctx.info.stmt).setFilePath(ctx.filePath.getText()); - ((ImportStatement) ctx.info.stmt).setNamespace(namespace); + setupContextInfo(ctx.info, namespace, filePath, ctx.filePath.getText(), prog); } } @@ -1470,12 +1443,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements @Override public void exitPathStatement(PathStatementContext ctx) { PathStatement stmt = new PathStatement(ctx.pathValue.getText()); - String filePath = ctx.pathValue.getText(); - if((filePath.startsWith("\"") && filePath.endsWith("\"")) || - filePath.startsWith("'") && filePath.endsWith("'")) { - filePath = filePath.substring(1, filePath.length()-1); - } - + String filePath = UtilFunctions.unquote(ctx.pathValue.getText()); _workingDir = filePath; ctx.info.stmt = stmt; } http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java index 1947c00..e32cb22 100644 --- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java @@ -533,9 +533,9 @@ public class UtilFunctions } public static String unquote(String s) { - if (s != null - && s.length() >=2 && ((s.startsWith("\"") && s.endsWith("\"")) - || (s.startsWith("'") && s.endsWith("'")))) { + if (s != null && s.length() >=2 + && ((s.startsWith("\"") && s.endsWith("\"")) + || (s.startsWith("'") && s.endsWith("'")))) { s = s.substring(1, s.length() - 1); } return s; http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/test/java/org/apache/sysml/test/integration/functions/jmlc/NamespaceFunctionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/NamespaceFunctionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/NamespaceFunctionTest.java new file mode 100644 index 0000000..8b65eb8 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/NamespaceFunctionTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.jmlc; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.sysml.api.jmlc.Connection; +import org.apache.sysml.api.jmlc.PreparedScript; +import org.apache.sysml.api.jmlc.ResultVariables; +import org.apache.sysml.runtime.io.IOUtilFunctions; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.junit.Assert; +import org.junit.Test; + +public class NamespaceFunctionTest extends AutomatedTestBase +{ + private final static String TEST_NAME1 = "foo.dml"; + private final static String TEST_NAME2 = "bar1.dml"; + private final static String TEST_NAME3 = "bar2.dml"; + + private final static String TEST_DIR = "functions/jmlc/"; + private final static String TEST_CLASS_DIR = TEST_DIR + NamespaceFunctionTest.class.getSimpleName() + "/"; + + private final static int rows = 700; + private final static int cols = 30; + private final static double sparsity1 = 0.7; + private final static double sparsity2 = 0.1; + + + @Override + public void setUp() { + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "F2" }) ); + } + + @Test + public void testJMLCNamespaceDense() throws IOException { + runJMLCNamespaceTest(false); + } + + @Test + public void testJMLCNamespaceSparse() throws IOException { + runJMLCNamespaceTest(true); + } + + + private void runJMLCNamespaceTest(boolean sparse) + throws IOException + { + TestConfiguration config = getTestConfiguration(TEST_NAME1); + loadTestConfiguration(config); + + //load scripts and create prepared script + Connection conn = new Connection(); + String script1 = conn.readScript(SCRIPT_DIR + TEST_DIR + TEST_NAME1); + Map<String,String> nsscripts = new HashMap<>(); + nsscripts.put(TEST_NAME2, conn.readScript(SCRIPT_DIR + TEST_DIR + TEST_NAME2)); + nsscripts.put(TEST_NAME3, conn.readScript(SCRIPT_DIR + TEST_DIR + TEST_NAME3)); + PreparedScript pstmt = conn.prepareScript(script1, + nsscripts, Collections.emptyMap(), new String[]{"X"}, new String[]{"Z"}, false); + + //generate input data + double sparsity = sparse ? sparsity2 : sparsity1; + MatrixBlock X = MatrixBlock.randOperations(rows, cols, sparsity, -1, 1, "uniform", 7); + + //execute script and get result + pstmt.setMatrix("X", X, true); + ResultVariables rs = pstmt.executeScript(); + MatrixBlock Z = rs.getMatrixBlock("Z"); + IOUtilFunctions.closeSilently(conn); + + //compare results + for(int i=0; i<rows; i++) + for(int j=0; j<cols; j++) + Assert.assertEquals(X.quickGetValue(i, j)+10, + Z.quickGetValue(i, j), 1e-10); + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/test/scripts/functions/jmlc/bar1.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/bar1.dml b/src/test/scripts/functions/jmlc/bar1.dml new file mode 100644 index 0000000..6b92e1a --- /dev/null +++ b/src/test/scripts/functions/jmlc/bar1.dml @@ -0,0 +1,27 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +exec = function(matrix[double] A) return (matrix[double] B) { + B = A; + for(i in 1:2) { + B = B + i; + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/test/scripts/functions/jmlc/bar2.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/bar2.dml b/src/test/scripts/functions/jmlc/bar2.dml new file mode 100644 index 0000000..58998d1 --- /dev/null +++ b/src/test/scripts/functions/jmlc/bar2.dml @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +source("bar1.dml") as ns3 + +exec = function(matrix[double] A) return (matrix[double] B) { + B = A + 1; + for(i in 1:2) { + B = ns3::exec(B); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/test/scripts/functions/jmlc/foo.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/foo.dml b/src/test/scripts/functions/jmlc/foo.dml new file mode 100644 index 0000000..b05c90b --- /dev/null +++ b/src/test/scripts/functions/jmlc/foo.dml @@ -0,0 +1,30 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +source("bar1.dml") as ns1 +source("bar2.dml") as ns2 + +X = read($1); + +Y = ns1::exec(X); +Z = ns2::exec(Y); + +write(Z, $2) http://git-wip-us.apache.org/repos/asf/systemml/blob/c250adab/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java index 6c0135c..9969cc0 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java @@ -40,6 +40,7 @@ import org.junit.runners.Suite; JMLCInputStreamReadTest.class, JMLCParfor2ForCompileTest.class, ReuseModelVariablesTest.class, + NamespaceFunctionTest.class, MulticlassSVMScoreTest.class })
