Repository: incubator-systemml Updated Branches: refs/heads/master fa73a1b85 -> 45c496c71
[SYSTEMML-651] Fix scalar input support for JMLC Add removeInput method to DataOp. If scalar read, call removeInput with iofilename in RewriteRemovePersistentReadWrite. Add JMLC scalar input test cases. Closes #129. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/45c496c7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/45c496c7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/45c496c7 Branch: refs/heads/master Commit: 45c496c7197b6ea832cedf6b59e8ff1826b69800 Parents: fa73a1b Author: Deron Eriksson <de...@us.ibm.com> Authored: Tue May 3 14:00:17 2016 -0700 Committer: Deron Eriksson <de...@us.ibm.com> Committed: Tue May 3 14:00:17 2016 -0700 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/DataOp.java | 29 +++- .../RewriteRemovePersistentReadWrite.java | 10 +- .../functions/jmlc/JMLCInputOutputTest.java | 141 +++++++++++++++++++ .../functions/jmlc/scalar-input-string.dml | 24 ++++ .../scripts/functions/jmlc/scalar-input.dml | 25 ++++ 5 files changed, 222 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/45c496c7/src/main/java/org/apache/sysml/hops/DataOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/DataOp.java b/src/main/java/org/apache/sysml/hops/DataOp.java index c907e33..a9cb8b9 100644 --- a/src/main/java/org/apache/sysml/hops/DataOp.java +++ b/src/main/java/org/apache/sysml/hops/DataOp.java @@ -19,20 +19,20 @@ package org.apache.sysml.hops; +import java.util.HashMap; +import java.util.Map.Entry; + import org.apache.sysml.conf.CompilerConfig.ConfigType; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.lops.Data; import org.apache.sysml.lops.Lop; -import org.apache.sysml.lops.LopsException; import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.lops.LopsException; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.util.LocalFileUtils; -import java.util.HashMap; -import java.util.Map.Entry; - public class DataOp extends Hop { @@ -568,4 +568,25 @@ public class DataOp extends Hop return ret; } + + /** + * Remove an input from the list of inputs and from the parameter index map. + * Parameter index map values higher than the index of the removed input + * will be decremented by one. + * + * @param inputName The name of the input to remove + */ + public void removeInput(String inputName) { + + int inputIndex = getParameterIndex(inputName); + _input.remove(inputIndex); + _paramIndexMap.remove(inputName); + + for (Entry<String, Integer> entry : _paramIndexMap.entrySet()) { + if (entry.getValue() > inputIndex) { + _paramIndexMap.put(entry.getKey(), (entry.getValue() - 1)); + } + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/45c496c7/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java index b908013..c71a935 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java @@ -27,8 +27,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.sysml.hops.DataOp; import org.apache.sysml.hops.Hop; import org.apache.sysml.hops.Hop.DataOpTypes; -import org.apache.sysml.hops.HopsException; import org.apache.sysml.hops.Hop.VisitStatus; +import org.apache.sysml.hops.HopsException; +import org.apache.sysml.parser.Expression.DataType; /** * This rewrite is a custom rewrite for JMLC in order to replace all persistent reads @@ -104,9 +105,12 @@ public class RewriteRemovePersistentReadWrite extends HopRewriteRule switch( dotype ) { case PERSISTENTREAD: - if( _inputs.contains(dop.getName()) ) + if( _inputs.contains(dop.getName()) ) { dop.setDataOpType(DataOpTypes.TRANSIENTREAD); - else + if (hop.getDataType() == DataType.SCALAR) { + dop.removeInput("iofilename"); + } + } else LOG.warn("Non-registered persistent read of variable '"+dop.getName()+"' (line "+dop.getBeginLine()+")."); break; case PERSISTENTWRITE: http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/45c496c7/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCInputOutputTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCInputOutputTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCInputOutputTest.java new file mode 100644 index 0000000..eee48b2 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCInputOutputTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.jmlc; + +import java.io.File; +import java.io.IOException; + +import org.apache.sysml.api.DMLException; +import org.apache.sysml.api.jmlc.Connection; +import org.apache.sysml.api.jmlc.PreparedScript; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.junit.Test; + +/** + * Test input and output capabilities of JMLC API. + * + */ +public class JMLCInputOutputTest extends AutomatedTestBase { + private final static String TEST_NAME = "JMLCInputOutputTest"; + private final static String TEST_DIR = "functions/jmlc/"; + + @Override + public void setUp() { + addTestConfiguration(TEST_DIR, TEST_NAME); + getAndLoadTestConfiguration(TEST_NAME); + } + + @Test + public void testScalarInputInt() throws IOException, DMLException { + Connection conn = new Connection(); + String str = conn.readScript(baseDirectory + File.separator + "scalar-input.dml"); + PreparedScript script = conn.prepareScript(str, new String[] { "inScalar1", "inScalar2" }, new String[] {}, + false); + int inScalar1 = 2; + int inScalar2 = 3; + script.setScalar("inScalar1", inScalar1); + script.setScalar("inScalar2", inScalar2); + + setExpectedStdOut("total:5"); + script.executeScript(); + } + + @Test + public void testScalarInputDouble() throws IOException, DMLException { + Connection conn = new Connection(); + String str = conn.readScript(baseDirectory + File.separator + "scalar-input.dml"); + PreparedScript script = conn.prepareScript(str, new String[] { "inScalar1", "inScalar2" }, new String[] {}, + false); + double inScalar1 = 3.5; + double inScalar2 = 4.5; + script.setScalar("inScalar1", inScalar1); + script.setScalar("inScalar2", inScalar2); + + setExpectedStdOut("total:8.0"); + script.executeScript(); + } + + // See: https://issues.apache.org/jira/browse/SYSTEMML-656 + // @Test + // public void testScalarInputBoolean() throws IOException, DMLException { + // Connection conn = new Connection(); + // String str = conn.readScript(baseDirectory + File.separator + + // "scalar-input.dml"); + // PreparedScript script = conn.prepareScript(str, new String[] { + // "inScalar1", "inScalar2" }, new String[] {}, + // false); + // boolean inScalar1 = true; + // boolean inScalar2 = true; + // script.setScalar("inScalar1", inScalar1); + // script.setScalar("inScalar2", inScalar2); + // + // setExpectedStdOut("total:TRUE"); + // script.executeScript(); + // } + + @Test + public void testScalarInputLong() throws IOException, DMLException { + Connection conn = new Connection(); + String str = conn.readScript(baseDirectory + File.separator + "scalar-input.dml"); + PreparedScript script = conn.prepareScript(str, new String[] { "inScalar1", "inScalar2" }, new String[] {}, + false); + long inScalar1 = 4; + long inScalar2 = 5; + script.setScalar("inScalar1", inScalar1); + script.setScalar("inScalar2", inScalar2); + + setExpectedStdOut("total:9"); + script.executeScript(); + } + + // See: https://issues.apache.org/jira/browse/SYSTEMML-658 + // @Test + // public void testScalarInputString() throws IOException, DMLException { + // Connection conn = new Connection(); + // String str = conn.readScript(baseDirectory + File.separator + + // "scalar-input.dml"); + // PreparedScript script = conn.prepareScript(str, new String[] { + // "inScalar1", "inScalar2" }, new String[] {}, + // false); + // String inScalar1 = "hello"; + // String inScalar2 = "goodbye"; + // script.setScalar("inScalar1", inScalar1); + // script.setScalar("inScalar2", inScalar2); + // + // setExpectedStdOut("total:hellogoodbye"); + // script.executeScript(); + // } + + @Test + public void testScalarInputStringExplicitValueType() throws IOException, DMLException { + Connection conn = new Connection(); + String str = conn.readScript(baseDirectory + File.separator + "scalar-input-string.dml"); + PreparedScript script = conn.prepareScript(str, new String[] { "inScalar1", "inScalar2" }, new String[] {}, + false); + String inScalar1 = "hello"; + String inScalar2 = "goodbye"; + script.setScalar("inScalar1", inScalar1); + script.setScalar("inScalar2", inScalar2); + + setExpectedStdOut("result:hellogoodbye"); + script.executeScript(); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/45c496c7/src/test/scripts/functions/jmlc/scalar-input-string.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/scalar-input-string.dml b/src/test/scripts/functions/jmlc/scalar-input-string.dml new file mode 100644 index 0000000..aadb9a1 --- /dev/null +++ b/src/test/scripts/functions/jmlc/scalar-input-string.dml @@ -0,0 +1,24 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +inScalar1 = read("./tmp/doesntexist1", data_type="scalar", value_type="string"); +inScalar2 = read("./tmp/doesntexist2", data_type="scalar", value_type="string"); +print("result:" + inScalar1 + inScalar2); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/45c496c7/src/test/scripts/functions/jmlc/scalar-input.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/scalar-input.dml b/src/test/scripts/functions/jmlc/scalar-input.dml new file mode 100644 index 0000000..9da112e --- /dev/null +++ b/src/test/scripts/functions/jmlc/scalar-input.dml @@ -0,0 +1,25 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +inScalar1 = read("./tmp/doesntexist1", data_type="scalar"); +inScalar2 = read("./tmp/doesntexist2", data_type="scalar"); +total = inScalar1 + inScalar2; +print("total:" + total); \ No newline at end of file
