This is an automated email from the ASF dual-hosted git repository. mboehm7 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/systemml.git
The following commit(s) were added to refs/heads/master by this push: new 3d876d3 [SYSTEMDS-412] Fix lineage-based reuse for update-inplace indexing 3d876d3 is described below commit 3d876d33ad019fe026799daaaaf540fc20a86ec4 Author: Matthias Boehm <mboe...@gmail.com> AuthorDate: Thu Jun 11 15:57:20 2020 +0200 [SYSTEMDS-412] Fix lineage-based reuse for update-inplace indexing This patch disabled lineage-based reuse for update-inplace left indexing operations as reuse would create incorrect results due to later in-place updates the change the cached data object. Furthermore, this patch also aims to make the codegen tests for robust wrt the surefire github action integration (less explain output). --- .../runtime/controlprogram/context/SparkExecutionContext.java | 8 ++------ .../java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java | 5 ++++- .../org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java | 2 +- .../apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java | 2 +- .../org/apache/sysds/test/functions/codegen/MiscPatternTest.java | 2 +- .../org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java | 2 +- .../apache/sysds/test/functions/codegen/OuterProdTmplTest.java | 2 +- .../org/apache/sysds/test/functions/codegen/RowAggTmplTest.java | 2 +- .../sysds/test/functions/codegen/RowConv2DOperationsTest.java | 2 +- .../sysds/test/functions/codegen/RowVectorComparisonTest.java | 2 +- .../apache/sysds/test/functions/codegen/SparseSideInputTest.java | 2 +- .../apache/sysds/test/functions/codegen/SumProductChainTest.java | 2 +- 12 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java index a1e2b92..11a4e93 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java @@ -171,18 +171,14 @@ public class SparkExecutionContext extends ExecutionContext _spctx = null; } - public void close() - { + public void close() { synchronized( SparkExecutionContext.class ) { - if( _spctx != null ) - { + if( _spctx != null ) { //stop the spark context if existing _spctx.stop(); - //make sure stopped context is never used again _spctx = null; } - } } diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java index 48a512a..22964ba 100644 --- a/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java +++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java @@ -25,6 +25,7 @@ import org.apache.sysds.runtime.controlprogram.context.ExecutionContext; import org.apache.sysds.runtime.instructions.Instruction; import org.apache.sysds.runtime.instructions.cp.ComputationCPInstruction; import org.apache.sysds.runtime.instructions.cp.ListIndexingCPInstruction; +import org.apache.sysds.runtime.instructions.cp.MatrixIndexingCPInstruction; import java.util.Comparator; @@ -151,7 +152,9 @@ public class LineageCacheConfig && !(inst instanceof ListIndexingCPInstruction); boolean rightop = (ArrayUtils.contains(REUSE_OPCODES, inst.getOpcode()) || (inst.getOpcode().equals("append") && isVectorAppend(inst, ec))); - return insttype && rightop; + boolean updateInplace = (inst instanceof MatrixIndexingCPInstruction) + && ec.getMatrixObject(((ComputationCPInstruction)inst).input1).getUpdateType().isInPlace(); + return insttype && rightop && !updateInplace; } private static boolean isVectorAppend(Instruction inst, ExecutionContext ec) { diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java index bd369c0..b1d184c 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java @@ -477,7 +477,7 @@ public class CellwiseTmplTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "-stats", "-args", output("S") }; + programArgs = new String[]{"-stats", "-args", output("S") }; fullRScriptName = HOME + testname + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java index c48db84..457285c 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java @@ -127,7 +127,7 @@ public class DAGCellwiseTmplTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "hops", "-stats", + programArgs = new String[]{"hops", "-stats", "-args", String.valueOf(cols), output("S") }; fullRScriptName = HOME + testname + ".R"; diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java index 60309b4..08c631c 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java @@ -126,7 +126,7 @@ public class MiscPatternTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "recompile_runtime", "-stats", "-args", output("S") }; + programArgs = new String[]{"recompile_runtime", "-stats", "-args", output("S") }; fullRScriptName = HOME + testname + ".R"; rCmd = getRCmd(expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java index d452313..d0b1c5a 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java @@ -174,7 +174,7 @@ public class MultiAggTmplTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "-stats", "-args", output("S") }; + programArgs = new String[]{"-stats", "-args", output("S") }; fullRScriptName = HOME + testname + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java index b4cdc96..d64f7ed 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java @@ -265,7 +265,7 @@ public class OuterProdTmplTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "-stats", "-args", output("S"), input("A")}; + programArgs = new String[]{"-stats", "-args", output("S"), input("A")}; fullRScriptName = HOME + testname + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java index 10a816c..695ee51 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java @@ -799,7 +799,7 @@ public class RowAggTmplTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "-stats", "-args", output("S") }; + programArgs = new String[]{"-stats", "-args", output("S") }; fullRScriptName = HOME + testname + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java index e76b1ab..c80508d 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java @@ -84,7 +84,7 @@ public class RowConv2DOperationsTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "recompile_runtime", "-stats", "-args", + programArgs = new String[]{"recompile_runtime", "-stats", "-args", String.valueOf(imgSize), String.valueOf(numImg), String.valueOf(numChannels), String.valueOf(numFilters), String.valueOf(filterSize), String.valueOf(stride), String.valueOf(pad), output("B"), sparseVal1, sparseVal2 }; diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java index cbbc9c3..c33a471 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java @@ -132,7 +132,7 @@ public class RowVectorComparisonTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "-stats", "-args", output("R") }; + programArgs = new String[]{"-stats", "-args", output("R") }; fullRScriptName = HOME + testname + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java index b9a2bb7..74460e4 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java @@ -153,7 +153,7 @@ public class SparseSideInputTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "-stats", "-args", + programArgs = new String[]{"-stats", "-args", input("X"), input("Y"), output("R") }; fullRScriptName = HOME + testname + ".R"; diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java index 663dcad..f674f88 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java @@ -110,7 +110,7 @@ public class SumProductChainTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "hops", "-stats", + programArgs = new String[]{"hops", "-stats", "-args", input("X"), output("R") }; fullRScriptName = HOME + testname + ".R";