Repository: systemml Updated Branches: refs/heads/master cffefca30 -> 5aadb4b22
[MINOR] Refactoring and cleanup CLI arguments parsing Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/5aadb4b2 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/5aadb4b2 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/5aadb4b2 Branch: refs/heads/master Commit: 5aadb4b2204b9238e73025c837a1aa3ed2908fe4 Parents: cffefca Author: Matthias Boehm <mboe...@gmail.com> Authored: Mon Jul 16 11:48:53 2018 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Mon Jul 16 11:48:53 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/sysml/api/DMLOptions.java | 273 +++++++ .../java/org/apache/sysml/api/DMLScript.java | 268 +------ .../sysml/api/mlcontext/ScriptExecutor.java | 2 +- .../sysml/test/unit/CLIOptionsParserTest.java | 728 +++++++++---------- 4 files changed, 627 insertions(+), 644 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/5aadb4b2/src/main/java/org/apache/sysml/api/DMLOptions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/DMLOptions.java b/src/main/java/org/apache/sysml/api/DMLOptions.java new file mode 100644 index 0000000..1f0f45a --- /dev/null +++ b/src/main/java/org/apache/sysml/api/DMLOptions.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.api; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.api.mlcontext.ScriptType; +import org.apache.sysml.hops.OptimizerUtils; +import org.apache.sysml.utils.Explain; +import org.apache.sysml.utils.Explain.ExplainType; + +/** + * Set of DMLOptions that can be set through the command line + * and {@link org.apache.sysml.api.mlcontext.MLContext} + * The values have been initialized with the default values + * Despite there being a DML and PyDML, this class is named DMLOptions + * to keep it consistent with {@link DMLOptions} and {@link DMLOptions} + */ +public class DMLOptions { + public final Options options; + public Map<String, String> argVals = new HashMap<>(); // Arguments map containing either named arguments or arguments by position for a DML program + public String configFile = null; // Path to config file if default config and default config is to be overriden + public boolean clean = false; // Whether to clean up all SystemML working directories (FS, DFS) + public boolean stats = false; // Whether to record and print the statistics + public int statsCount = 10; // Default statistics count + public boolean memStats = false; // max memory statistics + public Explain.ExplainType explainType = Explain.ExplainType.NONE; // Whether to print the "Explain" and if so, what type + public RUNTIME_PLATFORM execMode = OptimizerUtils.getDefaultExecutionMode(); // Execution mode standalone, MR, Spark or a hybrid + public boolean gpu = false; // Whether to use the GPU + public boolean forceGPU = false; // Whether to ignore memory & estimates and always use the GPU + public boolean debug = false; // to go into debug mode to be able to step through a program + public ScriptType scriptType = ScriptType.DML; // whether the script is a DML or PyDML script + public String filePath = null; // path to script + public String script = null; // the script itself + public boolean help = false; // whether to print the usage option + + public final static DMLOptions defaultOptions = new DMLOptions(null); + + public DMLOptions(Options opts) { + options = opts; + } + + @Override + public String toString() { + return "DMLOptions{" + + "argVals=" + argVals + + ", configFile='" + configFile + '\'' + + ", clean=" + clean + + ", stats=" + stats + + ", statsCount=" + statsCount + + ", memStats=" + memStats + + ", explainType=" + explainType + + ", execMode=" + execMode + + ", gpu=" + gpu + + ", forceGPU=" + forceGPU + + ", debug=" + debug + + ", scriptType=" + scriptType + + ", filePath='" + filePath + '\'' + + ", script='" + script + '\'' + + ", help=" + help + + '}'; + } + + + /** + * Parses command line arguments to create a {@link DMLOptions} instance with the correct options + * @param args arguments from the command line + * @param options an {@link Options} instance containing the options that need to be parsed + * @return an instance of {@link Options} that contain the correct {@link Option}s. + * @throws org.apache.commons.cli.ParseException if there is an incorrect option specified in the CLI + */ + public static DMLOptions parseCLArguments(String[] args) + throws org.apache.commons.cli.ParseException + { + Options options = createCLIOptions(); + CommandLineParser clParser = new PosixParser(); + CommandLine line = clParser.parse(options, args); + + DMLOptions dmlOptions = new DMLOptions(options); + dmlOptions.help = line.hasOption("help"); + dmlOptions.scriptType = line.hasOption("python") ? ScriptType.PYDML : ScriptType.DML; + dmlOptions.debug = line.hasOption("debug"); + dmlOptions.gpu = line.hasOption("gpu"); + if (dmlOptions.gpu) { + String force = line.getOptionValue("gpu"); + if (force != null) { + if (force.equalsIgnoreCase("force")) { + dmlOptions.forceGPU = true; + } else { + throw new org.apache.commons.cli.ParseException("Invalid argument specified for -gpu option"); + } + } + } + if (line.hasOption("exec")){ + String execMode = line.getOptionValue("exec"); + if (execMode != null){ + if (execMode.equalsIgnoreCase("hadoop")) dmlOptions.execMode = RUNTIME_PLATFORM.HADOOP; + else if (execMode.equalsIgnoreCase("singlenode")) dmlOptions.execMode = RUNTIME_PLATFORM.SINGLE_NODE; + else if (execMode.equalsIgnoreCase("hybrid")) dmlOptions.execMode = RUNTIME_PLATFORM.HYBRID; + else if (execMode.equalsIgnoreCase("hybrid_spark")) dmlOptions.execMode = RUNTIME_PLATFORM.HYBRID_SPARK; + else if (execMode.equalsIgnoreCase("spark")) dmlOptions.execMode = RUNTIME_PLATFORM.SPARK; + else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -exec option, must be one of [hadoop, singlenode, hybrid, hybrid_spark, spark]"); + } + } + if (line.hasOption("explain")) { + dmlOptions.explainType = ExplainType.RUNTIME; + String explainType = line.getOptionValue("explain"); + if (explainType != null){ + if (explainType.equalsIgnoreCase("hops")) dmlOptions.explainType = ExplainType.HOPS; + else if (explainType.equalsIgnoreCase("runtime")) dmlOptions.explainType = ExplainType.RUNTIME; + else if (explainType.equalsIgnoreCase("recompile_hops")) dmlOptions.explainType = ExplainType.RECOMPILE_HOPS; + else if (explainType.equalsIgnoreCase("recompile_runtime")) dmlOptions.explainType = ExplainType.RECOMPILE_RUNTIME; + else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime]"); + } + } + dmlOptions.stats = line.hasOption("stats"); + if (dmlOptions.stats){ + String statsCount = line.getOptionValue("stats"); + if (statsCount != null) { + try { + dmlOptions.statsCount = Integer.parseInt(statsCount); + } catch (NumberFormatException e) { + throw new org.apache.commons.cli.ParseException("Invalid argument specified for -stats option, must be a valid integer"); + } + } + } + dmlOptions.memStats = line.hasOption("mem"); + + dmlOptions.clean = line.hasOption("clean"); + + if (line.hasOption("config")){ + dmlOptions.configFile = line.getOptionValue("config"); + } + + if (line.hasOption("f")){ + dmlOptions.filePath = line.getOptionValue("f"); + } + + if (line.hasOption("s")){ + dmlOptions.script = line.getOptionValue("s"); + } + + // Positional arguments map is created as ("$1", "a"), ("$2", 123), etc + if (line.hasOption("args")){ + String[] argValues = line.getOptionValues("args"); + for (int k=0; k<argValues.length; k++){ + String str = argValues[k]; + if (!str.isEmpty()) { + dmlOptions.argVals.put("$" + (k+1), str); + } + } + } + + // Named arguments map is created as ("$K, 123), ("$X", "X.csv"), etc + if (line.hasOption("nvargs")){ + String varNameRegex = "^[a-zA-Z]([a-zA-Z0-9_])*$"; + String[] nvargValues = line.getOptionValues("nvargs"); + for (String str : nvargValues){ + if (!str.isEmpty()){ + String[] kv = str.split("="); + if (kv.length != 2){ + throw new org.apache.commons.cli.ParseException("Invalid argument specified for -nvargs option, must be a list of space separated K=V pairs, where K is a valid name of a variable in the DML/PyDML program"); + } + if (!kv[0].matches(varNameRegex)) { + throw new org.apache.commons.cli.ParseException("Invalid argument specified for -nvargs option, " + kv[0] + " does not seem like a valid variable name in DML. Valid variable names in DML start with upper-case or lower-case letter, and contain only letters, digits, or underscores"); + } + dmlOptions.argVals.put("$" + kv[0], kv[1]); + } + } + } + + return dmlOptions; + } + + /** + * Creates an {@link Options} instance for the command line parameters + * As of SystemML 0.13, Apache Commons CLI 1.2 is transitively in the classpath + * However the most recent version of Apache Commons CLI is 1.4 + * Creating CLI options is done using Static methods. Instead of {@link OptionBuilder}, + * CLI 1.4 uses Option.Builder which has non-static methods. + * @return an appropriate instance of {@link Options} + */ + @SuppressWarnings("static-access") + private static Options createCLIOptions() { + Options options = new Options(); + Option nvargsOpt = OptionBuilder.withArgName("key=value") + .withDescription("parameterizes DML script with named parameters of the form <key=value>; <key> should be a valid identifier in DML/PyDML") + .hasArgs().create("nvargs"); + Option argsOpt = OptionBuilder.withArgName("argN") + .withDescription("specifies positional parameters; first value will replace $1 in DML program; $2 will replace 2nd and so on") + .hasArgs().create("args"); + Option configOpt = OptionBuilder.withArgName("filename") + .withDescription("uses a given configuration file (can be on local/hdfs/gpfs; default values in SystemML-config.xml") + .hasArg().create("config"); + Option cleanOpt = OptionBuilder.withDescription("cleans up all SystemML working directories (FS, DFS); all other flags are ignored in this mode. \n") + .create("clean"); + Option statsOpt = OptionBuilder.withArgName("count") + .withDescription("monitors and reports summary execution statistics; heavy hitter <count> is 10 unless overridden; default off") + .hasOptionalArg().create("stats"); + Option memOpt = OptionBuilder.withDescription("monitors and reports max memory consumption in CP; default off") + .create("mem"); + Option explainOpt = OptionBuilder.withArgName("level") + .withDescription("explains plan levels; can be 'hops' / 'runtime'[default] / 'recompile_hops' / 'recompile_runtime'") + .hasOptionalArg().create("explain"); + Option execOpt = OptionBuilder.withArgName("mode") + .withDescription("sets execution mode; can be 'hadoop' / 'singlenode' / 'hybrid'[default] / 'hybrid_spark' / 'spark'") + .hasArg().create("exec"); + Option gpuOpt = OptionBuilder.withArgName("force") + .withDescription("uses CUDA instructions when reasonable; set <force> option to skip conservative memory estimates and use GPU wherever possible; default off") + .hasOptionalArg().create("gpu"); + Option debugOpt = OptionBuilder.withDescription("runs in debug mode; default off") + .create("debug"); + Option pythonOpt = OptionBuilder.withDescription("parses Python-like DML") + .create("python"); + Option fileOpt = OptionBuilder.withArgName("filename") + .withDescription("specifies dml/pydml file to execute; path can be local/hdfs/gpfs (prefixed with appropriate URI)") + .isRequired().hasArg().create("f"); + Option scriptOpt = OptionBuilder.withArgName("script_contents") + .withDescription("specified script string to execute directly") + .isRequired().hasArg().create("s"); + Option helpOpt = OptionBuilder.withDescription("shows usage message") + .create("help"); + + options.addOption(configOpt); + options.addOption(cleanOpt); + options.addOption(statsOpt); + options.addOption(memOpt); + options.addOption(explainOpt); + options.addOption(execOpt); + options.addOption(gpuOpt); + options.addOption(debugOpt); + options.addOption(pythonOpt); + + // Either a clean(-clean), a file(-f), a script(-s) or help(-help) needs to be specified + OptionGroup fileOrScriptOpt = new OptionGroup() + .addOption(scriptOpt).addOption(fileOpt).addOption(cleanOpt).addOption(helpOpt); + fileOrScriptOpt.setRequired(true); + options.addOptionGroup(fileOrScriptOpt); + + // Either -args or -nvargs + options.addOptionGroup(new OptionGroup() + .addOption(nvargsOpt).addOption(argsOpt)); + options.addOption(helpOpt); + + return options; + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/5aadb4b2/src/main/java/org/apache/sysml/api/DMLScript.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java index 9c8a3eb..50a23aa 100644 --- a/src/main/java/org/apache/sysml/api/DMLScript.java +++ b/src/main/java/org/apache/sysml/api/DMLScript.java @@ -31,20 +31,12 @@ import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Scanner; import org.apache.commons.cli.AlreadySelectedException; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.OptionGroup; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.PosixParser; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -117,55 +109,7 @@ public class DMLScript // ARC, // https://dbs.uni-leipzig.de/file/ARC.pdf // LOOP_AWARE // different policies for operations in for/while/parfor loop vs out-side the loop } - - /** - * Set of DMLOptions that can be set through the command line - * and {@link org.apache.sysml.api.mlcontext.MLContext} - * The values have been initialized with the default values - * Despite there being a DML and PyDML, this class is named DMLOptions - * to keep it consistent with {@link DMLScript} and {@link DMLOptions} - */ - public static class DMLOptions { - public Map<String, String> argVals = new HashMap<>(); // Arguments map containing either named arguments or arguments by position for a DML program - public String configFile = null; // Path to config file if default config and default config is to be overriden - public boolean clean = false; // Whether to clean up all SystemML working directories (FS, DFS) - public boolean stats = false; // Whether to record and print the statistics - public int statsCount = 10; // Default statistics count - public boolean memStats = false; // max memory statistics - public Explain.ExplainType explainType = Explain.ExplainType.NONE; // Whether to print the "Explain" and if so, what type - public DMLScript.RUNTIME_PLATFORM execMode = OptimizerUtils.getDefaultExecutionMode(); // Execution mode standalone, MR, Spark or a hybrid - public boolean gpu = false; // Whether to use the GPU - public boolean forceGPU = false; // Whether to ignore memory & estimates and always use the GPU - public boolean debug = false; // to go into debug mode to be able to step through a program - public ScriptType scriptType = ScriptType.DML; // whether the script is a DML or PyDML script - public String filePath = null; // path to script - public String script = null; // the script itself - public boolean help = false; // whether to print the usage option - - public final static DMLOptions defaultOptions = new DMLOptions(); - - @Override - public String toString() { - return "DMLOptions{" + - "argVals=" + argVals + - ", configFile='" + configFile + '\'' + - ", clean=" + clean + - ", stats=" + stats + - ", statsCount=" + statsCount + - ", memStats=" + memStats + - ", explainType=" + explainType + - ", execMode=" + execMode + - ", gpu=" + gpu + - ", forceGPU=" + forceGPU + - ", debug=" + debug + - ", scriptType=" + scriptType + - ", filePath='" + filePath + '\'' + - ", script='" + script + '\'' + - ", help=" + help + - '}'; - } - } - + public static RUNTIME_PLATFORM rtplatform = DMLOptions.defaultOptions.execMode; // the execution mode public static boolean STATISTICS = DMLOptions.defaultOptions.stats; // whether to print statistics public static boolean FINEGRAINED_STATISTICS = false; // whether to print fine-grained statistics @@ -258,202 +202,6 @@ public class DMLScript } /** - * Parses command line arguments to create a {@link DMLOptions} instance with the correct options - * @param args arguments from the command line - * @param options an {@link Options} instance containing the options that need to be parsed - * @return an instance of {@link Options} that contain the correct {@link Option}s. - * @throws org.apache.commons.cli.ParseException if there is an incorrect option specified in the CLI - */ - public static DMLOptions parseCLArguments(String[] args, Options options) throws org.apache.commons.cli.ParseException { - - CommandLineParser clParser = new PosixParser(); - CommandLine line = clParser.parse(options, args); - - DMLOptions dmlOptions = new DMLOptions(); - dmlOptions.help = line.hasOption("help"); - dmlOptions.scriptType = line.hasOption("python") ? ScriptType.PYDML : ScriptType.DML; - dmlOptions.debug = line.hasOption("debug"); - dmlOptions.gpu = line.hasOption("gpu"); - if (dmlOptions.gpu) { - String force = line.getOptionValue("gpu"); - if (force != null) { - if (force.equalsIgnoreCase("force")) { - dmlOptions.forceGPU = true; - } else { - throw new org.apache.commons.cli.ParseException("Invalid argument specified for -gpu option"); - } - } - } - if (line.hasOption("exec")){ - String execMode = line.getOptionValue("exec"); - if (execMode != null){ - if (execMode.equalsIgnoreCase("hadoop")) dmlOptions.execMode = RUNTIME_PLATFORM.HADOOP; - else if (execMode.equalsIgnoreCase("singlenode")) dmlOptions.execMode = RUNTIME_PLATFORM.SINGLE_NODE; - else if (execMode.equalsIgnoreCase("hybrid")) dmlOptions.execMode = RUNTIME_PLATFORM.HYBRID; - else if (execMode.equalsIgnoreCase("hybrid_spark")) dmlOptions.execMode = RUNTIME_PLATFORM.HYBRID_SPARK; - else if (execMode.equalsIgnoreCase("spark")) dmlOptions.execMode = RUNTIME_PLATFORM.SPARK; - else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -exec option, must be one of [hadoop, singlenode, hybrid, hybrid_spark, spark]"); - } - } - if (line.hasOption("explain")) { - dmlOptions.explainType = ExplainType.RUNTIME; - String explainType = line.getOptionValue("explain"); - if (explainType != null){ - if (explainType.equalsIgnoreCase("hops")) dmlOptions.explainType = ExplainType.HOPS; - else if (explainType.equalsIgnoreCase("runtime")) dmlOptions.explainType = ExplainType.RUNTIME; - else if (explainType.equalsIgnoreCase("recompile_hops")) dmlOptions.explainType = ExplainType.RECOMPILE_HOPS; - else if (explainType.equalsIgnoreCase("recompile_runtime")) dmlOptions.explainType = ExplainType.RECOMPILE_RUNTIME; - else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime]"); - } - } - dmlOptions.stats = line.hasOption("stats"); - if (dmlOptions.stats){ - String statsCount = line.getOptionValue("stats"); - if (statsCount != null) { - try { - dmlOptions.statsCount = Integer.parseInt(statsCount); - } catch (NumberFormatException e) { - throw new org.apache.commons.cli.ParseException("Invalid argument specified for -stats option, must be a valid integer"); - } - } - } - dmlOptions.memStats = line.hasOption("mem"); - - dmlOptions.clean = line.hasOption("clean"); - - if (line.hasOption("config")){ - dmlOptions.configFile = line.getOptionValue("config"); - } - - if (line.hasOption("f")){ - dmlOptions.filePath = line.getOptionValue("f"); - } - - if (line.hasOption("s")){ - dmlOptions.script = line.getOptionValue("s"); - } - - // Positional arguments map is created as ("$1", "a"), ("$2", 123), .... - if (line.hasOption("args")){ - String[] argValues = line.getOptionValues("args"); - for (int k=0; k<argValues.length; k++){ - String str = argValues[k]; - if (!str.isEmpty()) { - dmlOptions.argVals.put("$" + (k+1), str); - } - } - } - - // Named arguments map is created as ("$K, 123), ("$X", "X.csv"), .... - if (line.hasOption("nvargs")){ - String varNameRegex = "^[a-zA-Z]([a-zA-Z0-9_])*$"; - String[] nvargValues = line.getOptionValues("nvargs"); - for (String str : nvargValues){ - if (!str.isEmpty()){ - String[] kv = str.split("="); - if (kv.length != 2){ - throw new org.apache.commons.cli.ParseException("Invalid argument specified for -nvargs option, must be a list of space separated K=V pairs, where K is a valid name of a variable in the DML/PyDML program"); - } - if (!kv[0].matches(varNameRegex)) { - throw new org.apache.commons.cli.ParseException("Invalid argument specified for -nvargs option, " + kv[0] + " does not seem like a valid variable name in DML. Valid variable names in DML start with upper-case or lower-case letter, and contain only letters, digits, or underscores"); - } - dmlOptions.argVals.put("$" + kv[0], kv[1]); - } - } - } - - return dmlOptions; - - } - - /** - * Creates an {@link Options} instance for the command line parameters - * As of SystemML 0.13, Apache Commons CLI 1.2 is transitively in the classpath - * However the most recent version of Apache Commons CLI is 1.4 - * Creating CLI options is done using Static methods. This obviously makes it - * thread unsafe. Instead of {@link OptionBuilder}, CLI 1.4 uses Option.Builder which - * has non-static methods. - * @return an appropriate instance of {@link Options} - */ - @SuppressWarnings("static-access") - public static Options createCLIOptions() { - Options options = new Options(); - Option nvargsOpt = OptionBuilder.withArgName("key=value") - .withDescription("parameterizes DML script with named parameters of the form <key=value>; <key> should be a valid identifier in DML/PyDML") - .hasArgs() - .create("nvargs"); - Option argsOpt = OptionBuilder.withArgName("argN") - .withDescription("specifies positional parameters; first value will replace $1 in DML program; $2 will replace 2nd and so on") - .hasArgs() - .create("args"); - Option configOpt = OptionBuilder.withArgName("filename") - .withDescription("uses a given configuration file (can be on local/hdfs/gpfs; default values in SystemML-config.xml") - .hasArg() - .create("config"); - Option cleanOpt = OptionBuilder.withDescription("cleans up all SystemML working directories (FS, DFS); all other flags are ignored in this mode. \n") - .create("clean"); - Option statsOpt = OptionBuilder.withArgName("count") - .withDescription("monitors and reports summary execution statistics; heavy hitter <count> is 10 unless overridden; default off") - .hasOptionalArg() - .create("stats"); - Option memOpt = OptionBuilder.withDescription("monitors and reports max memory consumption in CP; default off") - .create("mem"); - Option explainOpt = OptionBuilder.withArgName("level") - .withDescription("explains plan levels; can be 'hops' / 'runtime'[default] / 'recompile_hops' / 'recompile_runtime'") - .hasOptionalArg() - .create("explain"); - Option execOpt = OptionBuilder.withArgName("mode") - .withDescription("sets execution mode; can be 'hadoop' / 'singlenode' / 'hybrid'[default] / 'hybrid_spark' / 'spark'") - .hasArg() - .create("exec"); - Option gpuOpt = OptionBuilder.withArgName("force") - .withDescription("uses CUDA instructions when reasonable; set <force> option to skip conservative memory estimates and use GPU wherever possible; default off") - .hasOptionalArg() - .create("gpu"); - Option debugOpt = OptionBuilder.withDescription("runs in debug mode; default off") - .create("debug"); - Option pythonOpt = OptionBuilder.withDescription("parses Python-like DML") - .create("python"); - Option fileOpt = OptionBuilder.withArgName("filename") - .withDescription("specifies dml/pydml file to execute; path can be local/hdfs/gpfs (prefixed with appropriate URI)") - .isRequired() - .hasArg() - .create("f"); - Option scriptOpt = OptionBuilder.withArgName("script_contents") - .withDescription("specified script string to execute directly") - .isRequired() - .hasArg() - .create("s"); - Option helpOpt = OptionBuilder.withDescription("shows usage message") - .create("help"); - - OptionGroup fileOrScriptOpt = new OptionGroup(); - // Either a clean(-clean), a file(-f), a script(-s) or help(-help) needs to be specified - fileOrScriptOpt.addOption(scriptOpt); - fileOrScriptOpt.addOption(fileOpt); - fileOrScriptOpt.addOption(cleanOpt); - fileOrScriptOpt.addOption(helpOpt); - fileOrScriptOpt.setRequired(true); - - OptionGroup argsOrNVArgsOpt = new OptionGroup(); - argsOrNVArgsOpt.addOption(nvargsOpt).addOption(argsOpt); // Either -args or -nvargs - - options.addOption(configOpt); - options.addOption(cleanOpt); - options.addOption(statsOpt); - options.addOption(memOpt); - options.addOption(explainOpt); - options.addOption(execOpt); - options.addOption(gpuOpt); - options.addOption(debugOpt); - options.addOption(pythonOpt); - options.addOptionGroup(fileOrScriptOpt); - options.addOptionGroup(argsOrNVArgsOpt); - options.addOption(helpOpt); - return options; - } - - /** * Single entry point for all public invocation alternatives (e.g., * main, executeScript, JaqlUdf etc) * @@ -461,16 +209,18 @@ public class DMLScript * @param args arguments * @return true if success, false otherwise */ + @SuppressWarnings("null") public static boolean executeScript( Configuration conf, String[] args ) { //parse arguments and set execution properties RUNTIME_PLATFORM oldrtplatform = rtplatform; //keep old rtplatform ExplainType oldexplain = EXPLAIN; //keep old explain - Options options = createCLIOptions(); + DMLOptions dmlOptions = null; + try { - DMLOptions dmlOptions = parseCLArguments(args, options); - + dmlOptions = DMLOptions.parseCLArguments(args); + STATISTICS = dmlOptions.stats; STATISTICS_COUNT = dmlOptions.statsCount; JMLC_MEM_STATISTICS = dmlOptions.memStats; @@ -489,7 +239,7 @@ public class DMLScript if (help) { HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp( "systemml", options ); + formatter.printHelp( "systemml", dmlOptions.options ); return true; } @@ -525,13 +275,13 @@ public class DMLScript catch(AlreadySelectedException e) { System.err.println("Mutually exclusive options were selected. " + e.getMessage()); HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp( "systemml", options ); + formatter.printHelp( "systemml", dmlOptions.options ); return false; } catch(org.apache.commons.cli.ParseException e) { System.err.println(e.getMessage()); HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp( "systemml", options ); + formatter.printHelp( "systemml", dmlOptions.options ); } catch (ParseException | DMLScriptException e) { throw e; http://git-wip-us.apache.org/repos/asf/systemml/blob/5aadb4b2/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java index f3303de..00fc096 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java @@ -26,7 +26,7 @@ import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.DMLOptions; +import org.apache.sysml.api.DMLOptions; import org.apache.sysml.api.DMLScript.EvictionPolicy; import org.apache.sysml.api.ScriptExecutorUtils; import org.apache.sysml.api.jmlc.JMLCUtils; http://git-wip-us.apache.org/repos/asf/systemml/blob/5aadb4b2/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java b/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java index 8018bcf..95b671a 100644 --- a/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java +++ b/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java @@ -1,18 +1,18 @@ /* * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ @@ -23,8 +23,8 @@ import java.util.Map; import org.apache.commons.cli.AlreadySelectedException; import org.apache.commons.cli.MissingOptionException; -import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.sysml.api.DMLOptions; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.mlcontext.ScriptType; import org.apache.sysml.utils.Explain; @@ -34,382 +34,342 @@ import org.junit.Test; public class CLIOptionsParserTest { - @Test(expected = MissingOptionException.class) - public void testNoOptions() throws Exception { - String cl = "systemml"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.parseCLArguments(args, options); - } - - @Test - public void testFile() throws Exception { - String cl = "systemml -f test.dml"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals("test.dml", o.filePath); - Assert.assertEquals(ScriptType.DML, o.scriptType); - - } - - @Test - public void testScript() throws Exception { - String cl = "systemml -s \"print('hello')\""; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals("print('hello')", o.script); - } - - @Test - public void testConfig() throws Exception { - String cl = "systemml -s \"print('hello')\" -config SystemML-config.xml"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals("print('hello')", o.script); - Assert.assertEquals("SystemML-config.xml", o.configFile); - } - - @Test - public void testDebug() throws Exception { - String cl = "systemml -s \"print('hello')\" -debug"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals("print('hello')", o.script); - Assert.assertEquals(true, o.debug); - } - - @Test - public void testClean() throws Exception { - String cl = "systemml -clean"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(true, o.clean); - } - - @Test(expected = AlreadySelectedException.class) - public void testBadClean() throws Exception { - String cl = "systemml -clean -f test.dml"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.parseCLArguments(args, options); - } - - @Test(expected = AlreadySelectedException.class) - public void testBadScript() throws Exception { - String cl = "systemml -f test.dml -s \"print('hello')\""; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.parseCLArguments(args, options); - } - - @Test - public void testStats() throws Exception { - String cl = "systemml -f test.dml -stats"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(true, o.stats); - Assert.assertEquals(10, o.statsCount); - } - - @Test - public void testStatsCount() throws Exception { - String cl = "systemml -f test.dml -stats 9123"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(true, o.stats); - Assert.assertEquals(9123, o.statsCount); - } - - @Test(expected = ParseException.class) - public void testBadStats() throws Exception { - String cl = "systemml -f test.dml -stats help"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(true, o.stats); - } - - @Test - public void testGPUForce() throws Exception { - String cl = "systemml -f test.dml -gpu force"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(true, o.gpu); - Assert.assertEquals(true, o.forceGPU); - } - - @Test(expected = ParseException.class) - public void testBadGPUOption() throws Exception { - String cl = "systemml -f test.dml -gpu f2orce"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.parseCLArguments(args, options); - } - - @Test - public void testPython() throws Exception { - String cl = "systemml -f test.dml -python"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(ScriptType.PYDML, o.scriptType); - } - - @Test - public void testHelp() throws Exception { - String cl = "systemml -help"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(true, o.help); - } - - @Test(expected = AlreadySelectedException.class) - public void testBadHelp() throws Exception { - String cl = "systemml -help -clean"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(true, o.help); - } - - @Test - public void testExplain1() throws Exception { - String cl = "systemml -f test.dml -explain"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(Explain.ExplainType.RUNTIME, o.explainType); - } - - @Test - public void testExplain2() throws Exception { - String cl = "systemml -f test.dml -explain hops"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(Explain.ExplainType.HOPS, o.explainType); - } - - @Test - public void testExplain3() throws Exception { - String cl = "systemml -f test.dml -explain runtime"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(Explain.ExplainType.RUNTIME, o.explainType); - } - - @Test - public void testExplain4() throws Exception { - String cl = "systemml -f test.dml -explain recompile_hops"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(Explain.ExplainType.RECOMPILE_HOPS, o.explainType); - } - - @Test - public void testExplain5() throws Exception { - String cl = "systemml -f test.dml -explain recompile_runtime"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(Explain.ExplainType.RECOMPILE_RUNTIME, o.explainType); - } - - @Test - public void testExec1() throws Exception { - String cl = "systemml -f test.dml -exec hadoop"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HADOOP, o.execMode); - } - - @Test - public void testExec2() throws Exception { - String cl = "systemml -f test.dml -exec spark"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.SPARK, o.execMode); - } - - @Test - public void testExec3() throws Exception { - String cl = "systemml -f test.dml -exec singlenode"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.SINGLE_NODE, o.execMode); - } - - @Test - public void testExec4() throws Exception { - String cl = "systemml -f test.dml -exec hybrid"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HYBRID, o.execMode); - } - - @Test - public void testExec5() throws Exception { - String cl = "systemml -f test.dml -exec hybrid_spark"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK, o.execMode); - } - - @Test(expected = ParseException.class) - public void testBadExec() throws Exception { - String cl = "systemml -f test.dml -exec new_system"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.parseCLArguments(args, options); - } - - @Test - public void testArgs1() throws Exception { - String cl = "systemml -f test.dml -args 10 \"x.csv\""; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Map<String, String> m = o.argVals; - Assert.assertEquals(2, m.size()); - Assert.assertEquals("10", m.get("$1")); - Assert.assertEquals("x.csv", m.get("$2")); - } - - @Test - public void testArgs2() throws Exception { - String cl = "systemml -f test.dml -args 10 \"x.csv\" 1234.2 systemml.conf -config systemml.conf"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Map<String, String> m = o.argVals; - Assert.assertEquals(4, m.size()); - Assert.assertEquals("10", m.get("$1")); - Assert.assertEquals("x.csv", m.get("$2")); - Assert.assertEquals("1234.2", m.get("$3")); - Assert.assertEquals("systemml.conf", m.get("$4")); - } - - @Test(expected = ParseException.class) - public void testBadArgs1() throws Exception { - String cl = "systemml -f test.dml -args -config systemml.conf"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.parseCLArguments(args, options); - } - - @Test - public void testNVArgs1() throws Exception { - String cl = "systemml -f test.dml -nvargs A=12 B=x.csv my123=12.2"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Map<String, String> m = o.argVals; - Assert.assertEquals(3, m.size()); - Assert.assertEquals("12", m.get("$A")); - Assert.assertEquals("x.csv", m.get("$B")); - Assert.assertEquals("12.2", m.get("$my123")); - } - - @Test(expected = ParseException.class) - public void testBadNVArgs1() throws Exception { - String cl = "systemml -f test.dml -nvargs"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.parseCLArguments(args, options); - } - - @Test(expected = ParseException.class) - public void testBadNVArgs2() throws Exception { - String cl = "systemml -f test.dml -nvargs asd qwe"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.parseCLArguments(args, options); - } - - @Test(expected = ParseException.class) - public void testBadNVArgs3() throws Exception { - String cl = "systemml -f test.dml -nvargs $X=12"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.parseCLArguments(args, options); - } - - @Test(expected = ParseException.class) - public void testBadNVArgs4() throws Exception { - String cl = "systemml -f test.dml -nvargs 123=123"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.parseCLArguments(args, options); - } - - /** - * For Apache Commons CLI, if an argument to an option is enclosed in quotes, - * the leading and trailing quotes are stripped away. For instance, if the options is -arg and the - * argument is "foo" - * -args "foo" - * Commons CLI will strip the quotes from "foo". This becomes troublesome when you really do - * want to pass in "foo" and not just foo. - * A way around this is to use 'foo` as done in {@link CLIOptionsParserTest#testNVArgs3()} - */ - @Test - public void testNVArgs2() throws Exception { - String cl = "systemml -f test.dml -args \"def\""; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Map<String, String> m = o.argVals; - Assert.assertEquals("def", m.get("$1")); - } - - - /** - * See comment in {@link CLIOptionsParserTest#testNVArgs2()} - */ - @Test - public void testNVArgs3() throws Exception { - String cl = "systemml -f test.dml -args 'def'"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Map<String, String> m = o.argVals; - Assert.assertEquals("'def'", m.get("$1")); - } - - /** - * See comment in {@link CLIOptionsParserTest#testNVArgs2()} - * Additionally, if we try to pass something like - * -nvargs X="foo" - * Commons CLI will strip the leading and trailing quotes (viz. double quotes), which - * causes it to return - * X="foo - * The way to overcome this is to enclose the <value> of the <key=value> pair in single quotes - * and strip them away in the parsing code ourselves. - * TODO: Read the javadoc for this method, we can add in this logic if required - */ - @Test - public void testNVArgs4() throws Exception { - String cl = "systemml -f test.dml -nvargs abc='def'"; - String[] args = cl.split(" "); - Options options = DMLScript.createCLIOptions(); - DMLScript.DMLOptions o = DMLScript.parseCLArguments(args, options); - Map<String, String> m = o.argVals; - Assert.assertEquals("'def'", m.get("$abc")); - } - + @Test(expected = MissingOptionException.class) + public void testNoOptions() throws Exception { + String cl = "systemml"; + String[] args = cl.split(" "); + DMLOptions.parseCLArguments(args); + } + + @Test + public void testFile() throws Exception { + String cl = "systemml -f test.dml"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals("test.dml", o.filePath); + Assert.assertEquals(ScriptType.DML, o.scriptType); + } + + @Test + public void testScript() throws Exception { + String cl = "systemml -s \"print('hello')\""; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals("print('hello')", o.script); + } + + @Test + public void testConfig() throws Exception { + String cl = "systemml -s \"print('hello')\" -config SystemML-config.xml"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals("print('hello')", o.script); + Assert.assertEquals("SystemML-config.xml", o.configFile); + } + + @Test + public void testDebug() throws Exception { + String cl = "systemml -s \"print('hello')\" -debug"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals("print('hello')", o.script); + Assert.assertEquals(true, o.debug); + } + + @Test + public void testClean() throws Exception { + String cl = "systemml -clean"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(true, o.clean); + } + + @Test(expected = AlreadySelectedException.class) + public void testBadClean() throws Exception { + String cl = "systemml -clean -f test.dml"; + String[] args = cl.split(" "); + DMLOptions.parseCLArguments(args); + } + + @Test(expected = AlreadySelectedException.class) + public void testBadScript() throws Exception { + String cl = "systemml -f test.dml -s \"print('hello')\""; + String[] args = cl.split(" "); + DMLOptions.parseCLArguments(args); + } + + @Test + public void testStats() throws Exception { + String cl = "systemml -f test.dml -stats"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(true, o.stats); + Assert.assertEquals(10, o.statsCount); + } + + @Test + public void testStatsCount() throws Exception { + String cl = "systemml -f test.dml -stats 9123"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(true, o.stats); + Assert.assertEquals(9123, o.statsCount); + } + + @Test(expected = ParseException.class) + public void testBadStats() throws Exception { + String cl = "systemml -f test.dml -stats help"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(true, o.stats); + } + + @Test + public void testGPUForce() throws Exception { + String cl = "systemml -f test.dml -gpu force"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(true, o.gpu); + Assert.assertEquals(true, o.forceGPU); + } + + @Test(expected = ParseException.class) + public void testBadGPUOption() throws Exception { + String cl = "systemml -f test.dml -gpu f2orce"; + String[] args = cl.split(" "); + DMLOptions.parseCLArguments(args); + } + + @Test + public void testPython() throws Exception { + String cl = "systemml -f test.dml -python"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(ScriptType.PYDML, o.scriptType); + } + + @Test + public void testHelp() throws Exception { + String cl = "systemml -help"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(true, o.help); + } + + @Test(expected = AlreadySelectedException.class) + public void testBadHelp() throws Exception { + String cl = "systemml -help -clean"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(true, o.help); + } + + @Test + public void testExplain1() throws Exception { + String cl = "systemml -f test.dml -explain"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(Explain.ExplainType.RUNTIME, o.explainType); + } + + @Test + public void testExplain2() throws Exception { + String cl = "systemml -f test.dml -explain hops"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(Explain.ExplainType.HOPS, o.explainType); + } + + @Test + public void testExplain3() throws Exception { + String cl = "systemml -f test.dml -explain runtime"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(Explain.ExplainType.RUNTIME, o.explainType); + } + + @Test + public void testExplain4() throws Exception { + String cl = "systemml -f test.dml -explain recompile_hops"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(Explain.ExplainType.RECOMPILE_HOPS, o.explainType); + } + + @Test + public void testExplain5() throws Exception { + String cl = "systemml -f test.dml -explain recompile_runtime"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(Explain.ExplainType.RECOMPILE_RUNTIME, o.explainType); + } + + @Test + public void testExec1() throws Exception { + String cl = "systemml -f test.dml -exec hadoop"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HADOOP, o.execMode); + } + + @Test + public void testExec2() throws Exception { + String cl = "systemml -f test.dml -exec spark"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.SPARK, o.execMode); + } + + @Test + public void testExec3() throws Exception { + String cl = "systemml -f test.dml -exec singlenode"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.SINGLE_NODE, o.execMode); + } + + @Test + public void testExec4() throws Exception { + String cl = "systemml -f test.dml -exec hybrid"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HYBRID, o.execMode); + } + + @Test + public void testExec5() throws Exception { + String cl = "systemml -f test.dml -exec hybrid_spark"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Assert.assertEquals(DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK, o.execMode); + } + + @Test(expected = ParseException.class) + public void testBadExec() throws Exception { + String cl = "systemml -f test.dml -exec new_system"; + String[] args = cl.split(" "); + DMLOptions.parseCLArguments(args); + } + + @Test + public void testArgs1() throws Exception { + String cl = "systemml -f test.dml -args 10 \"x.csv\""; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Map<String, String> m = o.argVals; + Assert.assertEquals(2, m.size()); + Assert.assertEquals("10", m.get("$1")); + Assert.assertEquals("x.csv", m.get("$2")); + } + + @Test + public void testArgs2() throws Exception { + String cl = "systemml -f test.dml -args 10 \"x.csv\" 1234.2 systemml.conf -config systemml.conf"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Map<String, String> m = o.argVals; + Assert.assertEquals(4, m.size()); + Assert.assertEquals("10", m.get("$1")); + Assert.assertEquals("x.csv", m.get("$2")); + Assert.assertEquals("1234.2", m.get("$3")); + Assert.assertEquals("systemml.conf", m.get("$4")); + } + + @Test(expected = ParseException.class) + public void testBadArgs1() throws Exception { + String cl = "systemml -f test.dml -args -config systemml.conf"; + String[] args = cl.split(" "); + DMLOptions.parseCLArguments(args); + } + + @Test + public void testNVArgs1() throws Exception { + String cl = "systemml -f test.dml -nvargs A=12 B=x.csv my123=12.2"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Map<String, String> m = o.argVals; + Assert.assertEquals(3, m.size()); + Assert.assertEquals("12", m.get("$A")); + Assert.assertEquals("x.csv", m.get("$B")); + Assert.assertEquals("12.2", m.get("$my123")); + } + + @Test(expected = ParseException.class) + public void testBadNVArgs1() throws Exception { + String cl = "systemml -f test.dml -nvargs"; + String[] args = cl.split(" "); + DMLOptions.parseCLArguments(args); + } + + @Test(expected = ParseException.class) + public void testBadNVArgs2() throws Exception { + String cl = "systemml -f test.dml -nvargs asd qwe"; + String[] args = cl.split(" "); + DMLOptions.parseCLArguments(args); + } + + @Test(expected = ParseException.class) + public void testBadNVArgs3() throws Exception { + String cl = "systemml -f test.dml -nvargs $X=12"; + String[] args = cl.split(" "); + DMLOptions.parseCLArguments(args); + } + + @Test(expected = ParseException.class) + public void testBadNVArgs4() throws Exception { + String cl = "systemml -f test.dml -nvargs 123=123"; + String[] args = cl.split(" "); + DMLOptions.parseCLArguments(args); + } + + /** + * For Apache Commons CLI, if an argument to an option is enclosed in quotes, + * the leading and trailing quotes are stripped away. For instance, if the options is -arg and the + * argument is "foo" + * -args "foo" + * Commons CLI will strip the quotes from "foo". This becomes troublesome when you really do + * want to pass in "foo" and not just foo. + * A way around this is to use 'foo` as done in {@link CLIOptionsParserTest#testNVArgs3()} + */ + @Test + public void testNVArgs2() throws Exception { + String cl = "systemml -f test.dml -args \"def\""; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Map<String, String> m = o.argVals; + Assert.assertEquals("def", m.get("$1")); + } + + + /** + * See comment in {@link CLIOptionsParserTest#testNVArgs2()} + */ + @Test + public void testNVArgs3() throws Exception { + String cl = "systemml -f test.dml -args 'def'"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Map<String, String> m = o.argVals; + Assert.assertEquals("'def'", m.get("$1")); + } + + /** + * See comment in {@link CLIOptionsParserTest#testNVArgs2()} + * Additionally, if we try to pass something like + * -nvargs X="foo" + * Commons CLI will strip the leading and trailing quotes (viz. double quotes), which + * causes it to return + * X="foo + * The way to overcome this is to enclose the <value> of the <key=value> pair in single quotes + * and strip them away in the parsing code ourselves. + * TODO: Read the javadoc for this method, we can add in this logic if required + */ + @Test + public void testNVArgs4() throws Exception { + String cl = "systemml -f test.dml -nvargs abc='def'"; + String[] args = cl.split(" "); + DMLOptions o = DMLOptions.parseCLArguments(args); + Map<String, String> m = o.argVals; + Assert.assertEquals("'def'", m.get("$abc")); + } } \ No newline at end of file
