Repository: systemml Updated Branches: refs/heads/master 317f2189c -> 47ce14fc6
[SYSTEMML-1932] Fix update-in-place compiler/runtime for frames, build This patch fixes update-in-place issues with leftindexing operations over frames. In detail, this includes additional data type checks in the rewrite for selecting update-in-place candidates, and a robustness improvement of the respective runtime to simply ignore candidates of wrong data types. Furthermore, this also includes an additional fix of several testsuite configuration files because our jenkins server expects a local scratch_space and local tmp dir per testcase. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/47ce14fc Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/47ce14fc Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/47ce14fc Branch: refs/heads/master Commit: 47ce14fc632303ef38675e1c9b274b93cafbeee2 Parents: 317f218 Author: Matthias Boehm <mboe...@gmail.com> Authored: Fri Sep 22 18:06:58 2017 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Fri Sep 22 18:34:33 2017 -0700 ---------------------------------------------------------------------- .../RewriteMarkLoopVariablesUpdateInPlace.java | 1 + .../org/apache/sysml/parser/DMLTranslator.java | 13 +---- .../runtime/controlprogram/ProgramBlock.java | 37 ++++++++------- .../context/ExecutionContext.java | 5 ++ .../TransformFrameEncodeColmapTest.java | 50 ++++++++++++++------ .../SystemML-config-codegen-compress.xml | 2 + .../codegen/SystemML-config-codegen.xml | 2 + .../codegen/SystemML-config-codegen6.xml | 2 + .../compress/SystemML-config-compress.xml | 2 + .../gdfo/SystemML-config-globalopt.xml | 2 + .../transform/TransformFrameEncodeColmap.dml | 39 --------------- .../transform/TransformFrameEncodeColmap1.dml | 39 +++++++++++++++ .../transform/TransformFrameEncodeColmap2.dml | 38 +++++++++++++++ 13 files changed, 150 insertions(+), 82 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/main/java/org/apache/sysml/hops/rewrite/RewriteMarkLoopVariablesUpdateInPlace.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteMarkLoopVariablesUpdateInPlace.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteMarkLoopVariablesUpdateInPlace.java index 9c6efb3..ce94ee1 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteMarkLoopVariablesUpdateInPlace.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteMarkLoopVariablesUpdateInPlace.java @@ -134,6 +134,7 @@ public class RewriteMarkLoopVariablesUpdateInPlace extends StatementBlockRewrite //valid if read/updated by leftindexing //CP exec type not evaluated here as no lops generated yet boolean validLix = hop instanceof DataOp + && hop.isMatrix() && hop.getInput().get(0).isMatrix() && hop.getInput().get(0) instanceof LeftIndexingOp && hop.getInput().get(0).getInput().get(0) instanceof DataOp && hop.getInput().get(0).getInput().get(0).getName().equals(varname); http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/main/java/org/apache/sysml/parser/DMLTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java index e9d4187..577af90 100644 --- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java @@ -2318,17 +2318,8 @@ public class DMLTranslator break; case WRITE: - String name = target.getName(); - currBuiltinOp = new DataOp( - target.getName(), target.getDataType(), target.getValueType(), DataOpTypes.PERSISTENTWRITE, hops.get(name), paramHops); - //MB: commented for dynamic write - /*Identifier ioFilename = (Identifier)source.getVarParam(DataExpression.IO_FILENAME); - if (!(ioFilename instanceof StringIdentifier)) { - LOG.error(source.printErrorLocation() + "processDataExpression():: Filename must be a constant string value"); - throw new ParseException(source.printErrorLocation() + "processDataExpression():: Filename must be a constant string value"); - } else { - ((DataOp)currBuiltinOp).setFileName(((StringIdentifier)ioFilename).getValue()); - }*/ + currBuiltinOp = new DataOp(target.getName(), target.getDataType(), target.getValueType(), + DataOpTypes.PERSISTENTWRITE, hops.get(target.getName()), paramHops); break; case RAND: http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java index b854ea0..c721ab5 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java @@ -309,25 +309,26 @@ public class ProgramBlock implements ParseInfo ArrayList<String> varnames = _sb.getUpdateInPlaceVars(); UpdateType[] flags = new UpdateType[varnames.size()]; - for( int i=0; i<flags.length; i++ ) - if( ec.getVariable(varnames.get(i)) != null ) { - String varname = varnames.get(i); - MatrixObject mo = ec.getMatrixObject(varname); - flags[i] = mo.getUpdateType(); - //create deep copy if required and if it fits in thread-local mem budget - if( flags[i]==UpdateType.COPY && OptimizerUtils.getLocalMemBudget()/2 > - OptimizerUtils.estimateSizeExactSparsity(mo.getMatrixCharacteristics())) { - MatrixObject moNew = new MatrixObject(mo); - MatrixBlock mbVar = mo.acquireRead(); - moNew.acquireModify( !mbVar.isInSparseFormat() ? new MatrixBlock(mbVar) : - new MatrixBlock(mbVar, MatrixBlock.DEFAULT_INPLACE_SPARSEBLOCK, true) ); - moNew.setFileName(mo.getFileName()+Lop.UPDATE_INPLACE_PREFIX+tid); - mo.release(); - moNew.release(); - moNew.setUpdateType(UpdateType.INPLACE); - ec.setVariable(varname, moNew); - } + for( int i=0; i<flags.length; i++ ) { + String varname = varnames.get(i); + if( !ec.isMatrixObject(varname) ) + continue; + MatrixObject mo = ec.getMatrixObject(varname); + flags[i] = mo.getUpdateType(); + //create deep copy if required and if it fits in thread-local mem budget + if( flags[i]==UpdateType.COPY && OptimizerUtils.getLocalMemBudget()/2 > + OptimizerUtils.estimateSizeExactSparsity(mo.getMatrixCharacteristics())) { + MatrixObject moNew = new MatrixObject(mo); + MatrixBlock mbVar = mo.acquireRead(); + moNew.acquireModify( !mbVar.isInSparseFormat() ? new MatrixBlock(mbVar) : + new MatrixBlock(mbVar, MatrixBlock.DEFAULT_INPLACE_SPARSEBLOCK, true) ); + moNew.setFileName(mo.getFileName()+Lop.UPDATE_INPLACE_PREFIX+tid); + mo.release(); + moNew.release(); + moNew.setUpdateType(UpdateType.INPLACE); + ec.setVariable(varname, moNew); } + } return flags; } http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java index 7df465c..b8adb5b 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java @@ -175,6 +175,11 @@ public class ExecutionContext { { return _variables.get(varname).getMetaData(); } + + public boolean isMatrixObject(String varname) { + Data dat = getVariable(varname); + return (dat!= null && dat instanceof MatrixObject); + } public MatrixObject getMatrixObject(String varname) throws DMLRuntimeException http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java index 5d19644..790f661 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java @@ -34,7 +34,9 @@ import org.junit.Test; public class TransformFrameEncodeColmapTest extends AutomatedTestBase { - private final static String TEST_NAME1 = "TransformFrameEncodeColmap"; + private final static String TEST_NAME1 = "TransformFrameEncodeColmap1"; + private final static String TEST_NAME2 = "TransformFrameEncodeColmap2"; + private final static String TEST_DIR = "functions/transform/"; private final static String TEST_CLASS_DIR = TEST_DIR + TransformFrameEncodeColmapTest.class.getSimpleName() + "/"; @@ -46,31 +48,51 @@ public class TransformFrameEncodeColmapTest extends AutomatedTestBase @Override public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "y" }) ); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "y" }) ); + addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "y" }) ); + } + + @Test + public void testHomesIDsSingleNode1() { + runTransformTest(TEST_NAME1, RUNTIME_PLATFORM.SINGLE_NODE, "csv", false); + } + + @Test + public void testHomesColnamesSingleNode1() { + runTransformTest(TEST_NAME1, RUNTIME_PLATFORM.SINGLE_NODE, "csv", true); + } + + @Test + public void testHomesIDsSpark1() { + runTransformTest(TEST_NAME1, RUNTIME_PLATFORM.SPARK, "csv", false); + } + + @Test + public void testHomesColnamesSpark1() { + runTransformTest(TEST_NAME1, RUNTIME_PLATFORM.SPARK, "csv", true); } @Test - public void testHomesIDsSingleNode() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false); + public void testHomesIDsSingleNode2() { + runTransformTest(TEST_NAME2, RUNTIME_PLATFORM.SINGLE_NODE, "csv", false); } @Test - public void testHomesColnamesSingleNode() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true); + public void testHomesColnamesSingleNode2() { + runTransformTest(TEST_NAME2, RUNTIME_PLATFORM.SINGLE_NODE, "csv", true); } @Test - public void testHomesIDsSpark() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false); + public void testHomesIDsSpark2() { + runTransformTest(TEST_NAME2, RUNTIME_PLATFORM.SPARK, "csv", false); } @Test - public void testHomesColnamesSpark() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true); + public void testHomesColnamesSpark2() { + runTransformTest(TEST_NAME2, RUNTIME_PLATFORM.SPARK, "csv", true); } - private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, boolean colnames ) + private void runTransformTest( String testname, RUNTIME_PLATFORM rt, String ofmt, boolean colnames ) { //set runtime platform RUNTIME_PLATFORM rtold = rtplatform; @@ -89,10 +111,10 @@ public class TransformFrameEncodeColmapTest extends AutomatedTestBase try { - getAndLoadTestConfiguration(TEST_NAME1); + getAndLoadTestConfiguration(testname); String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; + fullDMLScriptName = HOME + testname + ".dml"; programArgs = new String[]{"-explain","-nvargs", "DATA=" + HOME + "input/" + DATASET, "TFSPEC=" + HOME + "input/" + SPEC, http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/test/scripts/functions/codegen/SystemML-config-codegen-compress.xml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SystemML-config-codegen-compress.xml b/src/test/scripts/functions/codegen/SystemML-config-codegen-compress.xml index d8597fd..b3e56a7 100644 --- a/src/test/scripts/functions/codegen/SystemML-config-codegen-compress.xml +++ b/src/test/scripts/functions/codegen/SystemML-config-codegen-compress.xml @@ -18,6 +18,8 @@ --> <root> + <sysml.localtmpdir>/tmp/systemml</sysml.localtmpdir> + <sysml.scratch>scratch_space</sysml.scratch> <sysml.optlevel>7</sysml.optlevel> <sysml.compressed.linalg>true</sysml.compressed.linalg> <sysml.codegen.enabled>true</sysml.codegen.enabled> http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/test/scripts/functions/codegen/SystemML-config-codegen.xml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SystemML-config-codegen.xml b/src/test/scripts/functions/codegen/SystemML-config-codegen.xml index 3a00e70..d072ab3 100644 --- a/src/test/scripts/functions/codegen/SystemML-config-codegen.xml +++ b/src/test/scripts/functions/codegen/SystemML-config-codegen.xml @@ -18,6 +18,8 @@ --> <root> + <sysml.localtmpdir>/tmp/systemml</sysml.localtmpdir> + <sysml.scratch>scratch_space</sysml.scratch> <sysml.optlevel>7</sysml.optlevel> <sysml.codegen.enabled>true</sysml.codegen.enabled> <sysml.codegen.plancache>true</sysml.codegen.plancache> http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml b/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml index 35d0956..d7b6265 100644 --- a/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml +++ b/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml @@ -18,6 +18,8 @@ --> <root> + <sysml.localtmpdir>/tmp/systemml</sysml.localtmpdir> + <sysml.scratch>scratch_space</sysml.scratch> <sysml.optlevel>6</sysml.optlevel> <sysml.codegen.enabled>true</sysml.codegen.enabled> <sysml.codegen.plancache>true</sysml.codegen.plancache> http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/test/scripts/functions/compress/SystemML-config-compress.xml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/compress/SystemML-config-compress.xml b/src/test/scripts/functions/compress/SystemML-config-compress.xml index 9c4f4ba..24bdc7f 100644 --- a/src/test/scripts/functions/compress/SystemML-config-compress.xml +++ b/src/test/scripts/functions/compress/SystemML-config-compress.xml @@ -18,5 +18,7 @@ --> <root> + <sysml.localtmpdir>/tmp/systemml</sysml.localtmpdir> + <sysml.scratch>scratch_space</sysml.scratch> <sysml.compressed.linalg>true</sysml.compressed.linalg> </root> http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml b/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml index a59d797..01c4c9a 100644 --- a/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml +++ b/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml @@ -18,6 +18,8 @@ --> <root> + <sysml.localtmpdir>/tmp/systemml</sysml.localtmpdir> + <sysml.scratch>scratch_space</sysml.scratch> <sysml.optlevel>4</sysml.optlevel> <!-- piggybacked test for custom mapred/mapreduce configurations --> <mapreduce.task.io.sort.mb>50</mapreduce.task.io.sort.mb> http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/test/scripts/functions/transform/TransformFrameEncodeColmap.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformFrameEncodeColmap.dml b/src/test/scripts/functions/transform/TransformFrameEncodeColmap.dml deleted file mode 100644 index 98ea7f2..0000000 --- a/src/test/scripts/functions/transform/TransformFrameEncodeColmap.dml +++ /dev/null @@ -1,39 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - -F1 = read($DATA); -jspec = read($TFSPEC, data_type="scalar", value_type="string"); - -[X, M] = transformencode(target=F1, spec=jspec); -map = transformcolmap(target=M, spec=jspec); - -F2 = as.frame(matrix(0, nrow(F1), 1)); -for(i in 1:nrow(map)) { - T = X[, as.scalar(map[i,2]):as.scalar(map[i,3])]; - if( ncol(T)==1 ) - Tf = as.frame(T); - else - Tf = transformdecode(target=T, meta=M[,i], spec="{ids:true, dummycode:[1]}"); - F2 = cbind(F2, Tf); -} -F2 = F2[,2:ncol(F2)]; - -write(F2, $TFDATA, format=$OFMT); http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/test/scripts/functions/transform/TransformFrameEncodeColmap1.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformFrameEncodeColmap1.dml b/src/test/scripts/functions/transform/TransformFrameEncodeColmap1.dml new file mode 100644 index 0000000..98ea7f2 --- /dev/null +++ b/src/test/scripts/functions/transform/TransformFrameEncodeColmap1.dml @@ -0,0 +1,39 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +F1 = read($DATA); +jspec = read($TFSPEC, data_type="scalar", value_type="string"); + +[X, M] = transformencode(target=F1, spec=jspec); +map = transformcolmap(target=M, spec=jspec); + +F2 = as.frame(matrix(0, nrow(F1), 1)); +for(i in 1:nrow(map)) { + T = X[, as.scalar(map[i,2]):as.scalar(map[i,3])]; + if( ncol(T)==1 ) + Tf = as.frame(T); + else + Tf = transformdecode(target=T, meta=M[,i], spec="{ids:true, dummycode:[1]}"); + F2 = cbind(F2, Tf); +} +F2 = F2[,2:ncol(F2)]; + +write(F2, $TFDATA, format=$OFMT); http://git-wip-us.apache.org/repos/asf/systemml/blob/47ce14fc/src/test/scripts/functions/transform/TransformFrameEncodeColmap2.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformFrameEncodeColmap2.dml b/src/test/scripts/functions/transform/TransformFrameEncodeColmap2.dml new file mode 100644 index 0000000..1fb09c4 --- /dev/null +++ b/src/test/scripts/functions/transform/TransformFrameEncodeColmap2.dml @@ -0,0 +1,38 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +F1 = read($DATA); +jspec = read($TFSPEC, data_type="scalar", value_type="string"); + +[X, M] = transformencode(target=F1, spec=jspec); +map = transformcolmap(target=M, spec=jspec); + +F2 = F1; +for(i in 1:nrow(map)) { + T = X[, as.scalar(map[i,2]):as.scalar(map[i,3])]; + if( ncol(T)==1 ) + Tf = as.frame(T); + else + Tf = transformdecode(target=T, meta=M[,i], spec="{ids:true, dummycode:[1]}"); + F2[,i] = Tf; +} + +write(F2, $TFDATA, format=$OFMT);
