This is an automated email from the ASF dual-hosted git repository. baunsgaard pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 90bde55c800f170d142db40dd76e88856c4e70f4 Author: baunsgaard <[email protected]> AuthorDate: Sun Mar 14 00:19:35 2021 +0100 [MINOR] fix various tests - lib matrix DataGen test fix - to create a sparseBlock if null. - reduce nnzPropagation tests memory usage because sometime we have OOM fails of this test on GitHub - misc tests less debug output - cast to float in python tests - Compressed append Empty optimization --- src/main/java/org/apache/sysds/conf/DMLConfig.java | 2 +- .../hops/rewrite/RewriteCompressedReblock.java | 2 +- .../runtime/compress/CompressedMatrixBlock.java | 2 +- .../runtime/compress/colgroup/ColGroupEmpty.java | 9 +++ .../sysds/runtime/compress/lib/CLALibAppend.java | 72 ++++++++++++++++------ .../apache/sysds/runtime/data/SparseBlockCOO.java | 6 +- .../runtime/matrix/data/LibMatrixDatagen.java | 17 +++-- src/main/python/tests/matrix/test_print.py | 4 +- src/test/java/org/apache/sysds/test/TestUtils.java | 10 +-- .../test/functions/misc/AssertExpressionTest.java | 2 + .../functions/misc/ConditionalValidateTest.java | 2 + .../test/functions/misc/DataTypeCastingTest.java | 2 + .../test/functions/misc/DataTypeChangeTest.java | 2 + .../test/functions/misc/ExistsVariableTest.java | 4 +- .../functions/misc/FunctionInExpressionTest.java | 4 +- .../test/functions/misc/FunctionInliningTest.java | 2 + .../test/functions/misc/FunctionNamespaceTest.java | 4 ++ .../test/functions/misc/FunctionNotFoundTest.java | 2 + .../test/functions/misc/FunctionReturnTest.java | 6 +- ...nstantFoldingScalarVariablePropagationTest.java | 2 + .../functions/misc/IPADeadCodeEliminationTest.java | 2 + .../functions/misc/IPAFunctionInliningTest.java | 2 + .../functions/misc/IPALiteralReplacementTest.java | 2 + .../test/functions/misc/IPANnzPropagationTest.java | 2 + .../functions/misc/IPAScalarRecursionTest.java | 2 + .../misc/IPAScalarVariablePropagationTest.java | 4 ++ .../functions/misc/IPAUnknownRecursionTest.java | 2 + .../test/functions/misc/ListAndStructTest.java | 2 + .../test/functions/misc/ListAppendRemove.java | 2 + .../test/functions/misc/LongOverflowTest.java | 2 + .../functions/misc/NegativeLoopIncrementsTest.java | 4 +- .../test/functions/misc/NrowNcolStringTest.java | 2 + .../test/functions/misc/PrintExpressionTest.java | 2 + .../sysds/test/functions/misc/PrintMatrixTest.java | 2 + .../test/functions/misc/ReadAfterWriteTest.java | 2 + .../test/functions/misc/ScalarAssignmentTest.java | 2 + .../functions/misc/ZeroRowsColsMatrixTest.java | 2 + .../scripts/functions/misc/IPANnzPropagation1.dml | 2 +- .../scripts/functions/misc/IPANnzPropagation2.dml | 4 +- 39 files changed, 158 insertions(+), 42 deletions(-) diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java b/src/main/java/org/apache/sysds/conf/DMLConfig.java index 74ddf8a..5f90fb4 100644 --- a/src/main/java/org/apache/sysds/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java @@ -126,7 +126,7 @@ public class DMLConfig _defaultVals.put(CP_PARALLEL_IO, "true" ); _defaultVals.put(COMPRESSED_LINALG, Compression.CompressConfig.FALSE.name() ); _defaultVals.put(COMPRESSED_LOSSY, "false" ); - _defaultVals.put(COMPRESSED_VALID_COMPRESSIONS, "SDC,DDC,RLE,OLE"); + _defaultVals.put(COMPRESSED_VALID_COMPRESSIONS, "SDC,DDC"); _defaultVals.put(COMPRESSED_OVERLAPPING, "true" ); _defaultVals.put(COMPRESSED_SAMPLING_RATIO, "0.01"); _defaultVals.put(COMPRESSED_COCODE, "COST"); diff --git a/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java b/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java index 194a5d7..0f26fa1 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java @@ -215,7 +215,7 @@ public class RewriteCompressedReblock extends StatementBlockRewriteRule { private int numberCompressedOpsExecuted = 0; private int numberDecompressedOpsExecuted = 0; private int inefficientSupportedOpsExecuted = 0; - private int superEfficientSuportedOpsExecuted = 0; + // private int superEfficientSupportedOpsExecuted = 0; private boolean foundStart = false; private boolean usedInLoop = false; diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index 828610b..5dc44d1 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -1235,7 +1235,7 @@ public class CompressedMatrixBlock extends MatrixBlock { public MatrixBlock randOperationsInPlace(RandomMatrixGenerator rgen, Well1024a bigrand, long bSeed, int k) { throw new DMLRuntimeException("CompressedMatrixBlock: randOperationsInPlace not supported."); } - + @Override public MatrixBlock seqOperationsInPlace(double from, double to, double incr) { // output should always be uncompressed diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java index a70d250..7cda071 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java @@ -38,6 +38,7 @@ public class ColGroupEmpty extends ColGroupValue { super(); } + /** * Constructs an Constant Colum Group, that contains only one tuple, with the given value. * @@ -49,6 +50,14 @@ public class ColGroupEmpty extends ColGroupValue { _zeros = true; } + public static ColGroupEmpty generate(int nCol, int nRow){ + int[] cols = new int[nCol]; + for(int i =0; i < nCol; i++){ + cols[i] =i; + } + return new ColGroupEmpty(cols,nRow); + } + @Override public int[] getCounts(int[] out) { // nothing diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java index 2ea5397..bb130b6 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java @@ -20,6 +20,7 @@ package org.apache.sysds.runtime.compress.lib; import java.util.ArrayList; +import java.util.List; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.logging.Log; @@ -28,6 +29,7 @@ import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory; import org.apache.sysds.runtime.compress.CompressionStatistics; import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty; import org.apache.sysds.runtime.matrix.data.MatrixBlock; public class CLALibAppend { @@ -35,25 +37,25 @@ public class CLALibAppend { private static final Log LOG = LogFactory.getLog(CLALibAppend.class.getName()); public static MatrixBlock append(MatrixBlock left, MatrixBlock right) { - - // if(left.isEmpty()) - // return right; - // else if(right.isEmpty()) - // return left; + + if(left.isEmpty() && right instanceof CompressedMatrixBlock) + return appendLeftEmpty(left, (CompressedMatrixBlock) right); + else if(right.isEmpty() && left instanceof CompressedMatrixBlock) + return appendRightEmpty((CompressedMatrixBlock)left, right); + final int m = left.getNumRows(); final int n = left.getNumColumns() + right.getNumColumns(); - // try to compress both sides (if not already compressed). - if(!(left instanceof CompressedMatrixBlock) && m > 1000){ - LOG.info("Compressing left for append operation"); + if(!(left instanceof CompressedMatrixBlock) && m > 1000) { + LOG.warn("Compressing left for append operation"); Pair<MatrixBlock, CompressionStatistics> x = CompressedMatrixBlockFactory.compress(left); if(x.getRight().ratio > 3.0) left = x.getLeft(); - + } - if(!(right instanceof CompressedMatrixBlock) && m > 1000){ - LOG.info("Compressing right for append operation"); + if(!(right instanceof CompressedMatrixBlock) && m > 1000) { + LOG.warn("Compressing right for append operation"); Pair<MatrixBlock, CompressionStatistics> x = CompressedMatrixBlockFactory.compress(right); if(x.getRight().ratio > 3.0) right = x.getLeft(); @@ -69,25 +71,57 @@ public class CLALibAppend { // init result matrix CompressedMatrixBlock ret = new CompressedMatrixBlock(m, n); + ret = appendColGroups(ret, leftC.getColGroups(), rightC.getColGroups(), leftC.getNumColumns()); + return ret; + } + + private static MatrixBlock appendRightEmpty(CompressedMatrixBlock left, MatrixBlock right) { + + final int m = left.getNumRows(); + final int n = left.getNumColumns() + right.getNumColumns(); + CompressedMatrixBlock ret = new CompressedMatrixBlock(m,n); + + List<AColGroup> newGroup = new ArrayList<>(1); + newGroup.add(ColGroupEmpty.generate(right.getNumColumns(), right.getNumRows())); + ret = appendColGroups(ret, left.getColGroups(), newGroup, left.getNumColumns()); + + return ret; + } + + private static MatrixBlock appendLeftEmpty(MatrixBlock left, CompressedMatrixBlock right) { + final int m = left.getNumRows(); + final int n = left.getNumColumns() + right.getNumColumns(); + CompressedMatrixBlock ret = new CompressedMatrixBlock(m,n); + + List<AColGroup> newGroup = new ArrayList<>(1); + newGroup.add(ColGroupEmpty.generate(left.getNumColumns(), left.getNumRows())); + ret = appendColGroups(ret, newGroup, right.getColGroups(), left.getNumColumns()); + + return ret; + } + + private static CompressedMatrixBlock appendColGroups(CompressedMatrixBlock ret, List<AColGroup> left, + List<AColGroup> right, int leftNumCols) { + // shallow copy of lhs column groups - ret.allocateColGroupList(new ArrayList<AColGroup>(leftC.getColGroups().size() + rightC.getColGroups().size())); + ret.allocateColGroupList(new ArrayList<AColGroup>(left.size() + right.size())); - for(AColGroup group : leftC.getColGroups()){ + long nnz = 0; + for(AColGroup group : left) { AColGroup tmp = group.copy(); ret.getColGroups().add(tmp); + nnz += group.getNumberNonZeros(); } - for(AColGroup group : rightC.getColGroups()) { + for(AColGroup group : right) { AColGroup tmp = group.copy(); - tmp.shiftColIndices(left.getNumColumns()); + tmp.shiftColIndices(leftNumCols); ret.getColGroups().add(tmp); + nnz += group.getNumberNonZeros(); } - long nnzl = (leftC.getNonZeros() <= -1 ) ? leftC.recomputeNonZeros() : leftC.getNonZeros() ; - long nnzr = (rightC.getNonZeros() <= -1 ) ? rightC.recomputeNonZeros() : rightC.getNonZeros() ; - // meta data maintenance - ret.setNonZeros(nnzl + nnzr); + ret.setNonZeros(nnz); return ret; } diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java index aebf060..ce8e707 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java @@ -150,9 +150,9 @@ public class SparseBlockCOO extends SparseBlock //32B overhead per array, int/int/double arr in nnz double size = 16 + 8; //object + 2 int fields - size += MemoryEstimates.intArrayCost((int)lnnz); ; //rindexes array (row indexes) - size += MemoryEstimates.intArrayCost((int) lnnz); ; //cindexes array (column indexes) - size += MemoryEstimates.doubleArrayCost((int) lnnz);; //values array (non-zero values) + size += MemoryEstimates.intArrayCost((int)lnnz); //rindexes array (row indexes) + size += MemoryEstimates.intArrayCost((int) lnnz); //cindexes array (column indexes) + size += MemoryEstimates.doubleArrayCost((int) lnnz); //values array (non-zero values) //robustness for long overflows return (long) Math.min(size, Long.MAX_VALUE); diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java index a8fce54..2f1a51f 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java @@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.commons.math3.random.Well1024a; import org.apache.sysds.hops.DataGenOp; import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.controlprogram.parfor.util.IDSequence; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; @@ -186,6 +187,9 @@ public class LibMatrixDatagen int blen = rgen._blocksize; double sparsity = rgen._sparsity; + if(out instanceof CompressedMatrixBlock) + throw new DMLRuntimeException("Invalid to use compressed matrix block as output"); + // sanity check valid dimensions and sparsity checkMatrixDimensionsAndSparsity(rows, cols, sparsity); @@ -258,6 +262,9 @@ public class LibMatrixDatagen int blen = rgen._blocksize; double sparsity = rgen._sparsity; + if(out instanceof CompressedMatrixBlock) + throw new DMLRuntimeException("Invalid to use compressed matrix block as output"); + //sanity check valid dimensions and sparsity checkMatrixDimensionsAndSparsity(rows, cols, sparsity); @@ -496,13 +503,15 @@ public class LibMatrixDatagen // are always selected uniformly at random. nnzPRNG.setSeed(seed); - // block-level sparsity, which may differ from overall sparsity in the matrix. - // (e.g., border blocks may fall under skinny matrix turn point, in CP this would be - // irrelevant but we need to ensure consistency with MR) boolean localSparse = MatrixBlock.evalSparseFormatInMemory( blockrows, blockcols, (long)(sparsity*blockrows*blockcols)); - if ( localSparse ) { + if ( localSparse) { + SparseBlock c = out.sparseBlock; + if(c == null){ + out.allocateSparseRowsBlock(); + c = out.sparseBlock; + } // Prob [k-1 zeros before a nonzero] = Prob [k-1 < log(uniform)/log(1-p) < k] = p*(1-p)^(k-1), where p=sparsity double log1mp = Math.log(1-sparsity); int idx = 0; // takes values in range [1, blen*blen] (both ends including) diff --git a/src/main/python/tests/matrix/test_print.py b/src/main/python/tests/matrix/test_print.py index 670ce37..4dc5a4b 100644 --- a/src/main/python/tests/matrix/test_print.py +++ b/src/main/python/tests/matrix/test_print.py @@ -46,11 +46,11 @@ class TestPrint(unittest.TestCase): def test_print_01(self): Matrix(self.sds, np.array([1])).to_string().print().compute() - self.assertEqual('1.000',self.sds.get_stdout()[0]) + self.assertEqual(1,float(self.sds.get_stdout()[0])) def test_print_02(self): self.sds.scalar(1).print().compute() - self.assertEqual('1', self.sds.get_stdout()[0]) + self.assertEqual(1,float(self.sds.get_stdout()[0])) if __name__ == "__main__": unittest.main(exit=False) diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index 6244833..48ba4ac 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -1047,10 +1047,12 @@ public class TestUtils if (!compareCellValue(v1, v2, 0, ignoreNaN)) { if (!compareCellValue(v1, v2, tolerance, ignoreNaN)) { countErrorWithinTolerance++; - if(!flag) - System.out.println(e.getKey()+": "+v1+" <--> "+v2); - else - System.out.println(e.getKey()+": "+v2+" <--> "+v1); + if(LOG.isDebugEnabled()){ + if(!flag) + LOG.debug(e.getKey()+": "+v1+" <--> "+v2); + else + LOG.debug(e.getKey()+": "+v2+" <--> "+v1); + } } } else { countIdentical++; diff --git a/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java index 7fadbe7..1ee0e4f 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java @@ -58,6 +58,7 @@ public class AssertExpressionTest extends AutomatedTestBase */ private void runPrintExpressionTest( String testname, boolean rewrites ) { + setOutputBuffering(true); String TEST_NAME = testname; TestConfiguration config = getTestConfiguration(TEST_NAME); loadTestConfiguration(config); @@ -82,5 +83,6 @@ public class AssertExpressionTest extends AutomatedTestBase { OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldRewriteFlag; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java index b911a4a..bec66c3 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java @@ -102,6 +102,7 @@ public class ConditionalValidateTest extends AutomatedTestBase private void runTest( String testName, Class<?> exceptionClass, boolean fileExists ) { + setOutputBuffering(true); String TEST_NAME = testName; try @@ -135,5 +136,6 @@ public class ConditionalValidateTest extends AutomatedTestBase catch(Exception ex) { throw new RuntimeException(ex); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java index 6df7e9c..d524f64 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java @@ -87,6 +87,7 @@ public class DataTypeCastingTest extends AutomatedTestBase */ private void runTest( String testName, boolean matrixInput, Class<?> exceptionClass ) { + setOutputBuffering(true); String TEST_NAME = testName; int numVals = (exceptionClass != null ? 7 : 1); @@ -134,5 +135,6 @@ public class DataTypeCastingTest extends AutomatedTestBase { throw new RuntimeException(ex); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java index d50eb23..3b4741a 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java @@ -150,6 +150,7 @@ public class DataTypeChangeTest extends AutomatedTestBase private void runTest( String testName, Class<?> exceptionExpected ) { + setOutputBuffering(true); String RI_HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = RI_HOME + testName + ".dml"; programArgs = new String[]{}; @@ -159,6 +160,7 @@ public class DataTypeChangeTest extends AutomatedTestBase //integration test from outside SystemDS runTest(true, exceptionExpected != null, exceptionExpected, -1); + setOutputBuffering(false); } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java index dee93cd..9d229fb 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java @@ -63,12 +63,13 @@ public class ExistsVariableTest extends AutomatedTestBase } private void runExistsTest(String testName, boolean pos) { + setOutputBuffering(true); TestConfiguration config = getTestConfiguration(testName); loadTestConfiguration(config); String HOME = SCRIPT_DIR + TEST_DIR; String param = pos ? "1" : "0"; fullDMLScriptName = HOME + testName + ".dml"; - programArgs = new String[]{"-stats", "-args", param, output("R") }; + programArgs = new String[]{"-args", param, output("R") }; //run script and compare output runTest(true, false, null, -1); @@ -78,5 +79,6 @@ public class ExistsVariableTest extends AutomatedTestBase val = (val!=null) ? val : 0; Assert.assertTrue("Wrong result: "+param+" vs "+val, val==Double.parseDouble(param)); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java index 7c5cd44..6d0b260 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java @@ -89,12 +89,13 @@ public class FunctionInExpressionTest extends AutomatedTestBase private void runFunInExpressionTest( String testName ) { + setOutputBuffering(true); TestConfiguration config = getTestConfiguration(testName); loadTestConfiguration(config); String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testName + ".dml"; - programArgs = new String[]{"-stats", "-args", output("R") }; + programArgs = new String[]{"-args", output("R") }; fullRScriptName = HOME + testName + ".R"; rCmd = getRCmd(expectedDir()); @@ -105,5 +106,6 @@ public class FunctionInExpressionTest extends AutomatedTestBase //compare results double val = readDMLMatrixFromOutputDir("R").get(new CellIndex(1,1)); Assert.assertTrue("Wrong result: 7 vs "+val, Math.abs(val-7)<Math.pow(10, -13)); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionInliningTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionInliningTest.java index 8861d27..b539328 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionInliningTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionInliningTest.java @@ -85,6 +85,7 @@ public class FunctionInliningTest extends AutomatedTestBase private void runInliningTest( String testname, boolean IPA ) { + setOutputBuffering(true); boolean oldIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try @@ -125,6 +126,7 @@ public class FunctionInliningTest extends AutomatedTestBase { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldIPA; } + setOutputBuffering(false); } } \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionNamespaceTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionNamespaceTest.java index e8b016b..f3f3fe9 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionNamespaceTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionNamespaceTest.java @@ -161,6 +161,7 @@ public class FunctionNamespaceTest extends AutomatedTestBase private void runFunctionNamespaceTest(String TEST_NAME) { + setOutputBuffering(true); getAndLoadTestConfiguration(TEST_NAME); fullDMLScriptName = SCRIPT_DIR + TEST_DIR + TEST_NAME + ".dml"; @@ -183,10 +184,12 @@ public class FunctionNamespaceTest extends AutomatedTestBase finally { System.setErr(origStdErr); } + setOutputBuffering(false); } private void runFunctionNoInliningNamespaceTest(String TEST_NAME, boolean IPA) { + setOutputBuffering(true); boolean origIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; getAndLoadTestConfiguration(TEST_NAME); @@ -229,5 +232,6 @@ public class FunctionNamespaceTest extends AutomatedTestBase System.setErr(originalStdErr); OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = origIPA; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionNotFoundTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionNotFoundTest.java index 4c56806..c7ed903 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionNotFoundTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionNotFoundTest.java @@ -50,6 +50,7 @@ public class FunctionNotFoundTest extends AutomatedTestBase { } private void runFunctionNotFoundTest(String testName, Class<?> error) { + setOutputBuffering(true); TestConfiguration config = getTestConfiguration(testName); loadTestConfiguration(config); @@ -58,5 +59,6 @@ public class FunctionNotFoundTest extends AutomatedTestBase { programArgs = new String[] {}; runTest(true, true, error, -1); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java index 60fc339..1dd9d62 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java @@ -58,6 +58,8 @@ public class FunctionReturnTest extends AutomatedTestBase } private void runFunctionReturnTest( String testname, boolean IPA ) { + + setOutputBuffering(true); boolean oldIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA; try { @@ -66,12 +68,14 @@ public class FunctionReturnTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain"}; + programArgs = new String[]{}; runTest(true, false, null, -1); } finally { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldIPA; } + + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java index 74b60d5..8d2213c 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java @@ -94,6 +94,7 @@ public class IPAConstantFoldingScalarVariablePropagationTest extends AutomatedTe */ private void runIPAScalarVariablePropagationTest(String testname, boolean IPA_SECOND_CHANCE) { + setOutputBuffering(true); // Save old settings int oldIPANumRep = OptimizerUtils.IPA_NUM_REPETITIONS; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; @@ -125,5 +126,6 @@ public class IPAConstantFoldingScalarVariablePropagationTest extends AutomatedTe DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; rtplatform = platformOld; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java index daf034a..35a4d34 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java @@ -87,6 +87,7 @@ public class IPADeadCodeEliminationTest extends AutomatedTestBase private void runIPALiteralReplacementTest( String testname, boolean IPA ) { + setOutputBuffering(true); boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try { @@ -106,5 +107,6 @@ public class IPADeadCodeEliminationTest extends AutomatedTestBase finally { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java index ca9d156..83c90ca 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java @@ -122,6 +122,7 @@ public class IPAFunctionInliningTest extends AutomatedTestBase private void runIPAFunInlineTest( String testName, boolean IPA ) { + setOutputBuffering(true); boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try @@ -162,5 +163,6 @@ public class IPAFunctionInliningTest extends AutomatedTestBase finally { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java index dc832ca..1a96f52 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java @@ -80,6 +80,7 @@ public class IPALiteralReplacementTest extends AutomatedTestBase */ private void runIPALiteralReplacementTest( String testname, boolean IPA ) { + setOutputBuffering(true); boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try @@ -107,5 +108,6 @@ public class IPALiteralReplacementTest extends AutomatedTestBase { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA; } + setOutputBuffering(false); } } \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java index a57fee2..ffbaf23 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java @@ -54,6 +54,7 @@ public class IPANnzPropagationTest extends AutomatedTestBase private void runIPANnzPropgationTest(String testname) { + setOutputBuffering(true); // Save old settings boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; ExecMode platformOld = rtplatform; @@ -79,5 +80,6 @@ public class IPANnzPropagationTest extends AutomatedTestBase DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; rtplatform = platformOld; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java index c89e7db..a216cb8 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java @@ -42,6 +42,7 @@ public class IPAScalarRecursionTest extends AutomatedTestBase @Test public void testScalarRecursion() { + setOutputBuffering(true); String TEST_NAME = TEST_NAME1; try @@ -59,5 +60,6 @@ public class IPAScalarRecursionTest extends AutomatedTestBase { throw new RuntimeException(ex); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java index 788e1da..533d945 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java @@ -61,6 +61,8 @@ public class IPAScalarVariablePropagationTest extends AutomatedTestBase */ private void runIPAScalarVariablePropagationTest( String testname, boolean IPA ) { + + setOutputBuffering(true); boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try @@ -87,5 +89,7 @@ public class IPAScalarVariablePropagationTest extends AutomatedTestBase finally { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA; } + + setOutputBuffering(false); } } \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java index 24a628d..4d0c180 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java @@ -63,6 +63,7 @@ public class IPAUnknownRecursionTest extends AutomatedTestBase */ private void runIPAUnknownRecursionTest( boolean IPA ) { + setOutputBuffering(true); boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try @@ -91,5 +92,6 @@ public class IPAUnknownRecursionTest extends AutomatedTestBase { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA; } + setOutputBuffering(false); } } \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java index 4f7c31a..03598f3 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java @@ -163,6 +163,7 @@ public class ListAndStructTest extends AutomatedTestBase private void runListStructTest(String testname, boolean rewrites) { + setOutputBuffering(true); boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; try { @@ -193,5 +194,6 @@ public class ListAndStructTest extends AutomatedTestBase finally { OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java b/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java index 946ef67..aa83b4f 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java @@ -96,6 +96,7 @@ public class ListAppendRemove extends AutomatedTestBase private void runListAppendRemove(String testname, ExecType type, boolean rewrites, boolean conditional) { + setOutputBuffering(true); Types.ExecMode platformOld = setExecMode(type); boolean rewriteOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; @@ -132,5 +133,6 @@ public class ListAppendRemove extends AutomatedTestBase rtplatform = platformOld; OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewriteOld; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java b/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java index 5423f98..e16cc3d 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java @@ -92,6 +92,7 @@ public class LongOverflowTest extends AutomatedTestBase */ private void runOverflowTest( String testscript, boolean error ) { + setOutputBuffering(true); String TEST_NAME = testscript; try @@ -113,5 +114,6 @@ public class LongOverflowTest extends AutomatedTestBase { throw new RuntimeException(ex); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java b/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java index 7ec6db2..4ab4063 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java @@ -21,12 +21,12 @@ package org.apache.sysds.test.functions.misc; import java.util.HashMap; -import org.junit.Test; import org.apache.sysds.runtime.matrix.data.MatrixValue.CellIndex; import org.apache.sysds.runtime.meta.MatrixCharacteristics; import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; +import org.junit.Test; /** * @@ -91,6 +91,7 @@ public class NegativeLoopIncrementsTest extends AutomatedTestBase private void runNegativeLoopIncrementsTest( String testname, boolean vect, boolean multiStep ) { + setOutputBuffering(true); String TEST_NAME = testname; TestConfiguration config = getTestConfiguration(TEST_NAME); loadTestConfiguration(config); @@ -118,5 +119,6 @@ public class NegativeLoopIncrementsTest extends AutomatedTestBase //check meta data checkDMLMetaDataFile("R", new MatrixCharacteristics(1,1,1,1)); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java b/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java index 4a5c503..cd0865e 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java @@ -69,6 +69,7 @@ public class NrowNcolStringTest extends AutomatedTestBase */ private void runNxxStringTest( String testName ) { + setOutputBuffering(true); String TEST_NAME = testName; try @@ -86,5 +87,6 @@ public class NrowNcolStringTest extends AutomatedTestBase { throw new RuntimeException(ex); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java index b05e5dd..a4ab57c 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java @@ -70,6 +70,7 @@ public class PrintExpressionTest extends AutomatedTestBase */ private void runPrintExpressionTest( String testname, boolean rewrites ) { + setOutputBuffering(true); String TEST_NAME = testname; TestConfiguration config = getTestConfiguration(TEST_NAME); loadTestConfiguration(config); @@ -94,5 +95,6 @@ public class PrintExpressionTest extends AutomatedTestBase { OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldRewriteFlag; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java index 9b38e37..a9e3038 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java @@ -45,6 +45,7 @@ public class PrintMatrixTest extends AutomatedTestBase private void runTest( String testName, boolean exceptionExpected ) { + setOutputBuffering(true); TestConfiguration config = getTestConfiguration(TEST_NAME1); loadTestConfiguration(config); @@ -54,5 +55,6 @@ public class PrintMatrixTest extends AutomatedTestBase //run tests runTest(true, exceptionExpected, DMLException.class, -1); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java index fdf2bbc..1b2f609 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java @@ -105,6 +105,7 @@ public class ReadAfterWriteTest extends AutomatedTestBase */ private void runReadAfterWriteTest( String testName, boolean positive ) { + setOutputBuffering(true); String TEST_NAME = testName; try @@ -134,5 +135,6 @@ public class ReadAfterWriteTest extends AutomatedTestBase //cleanup TestUtils.clearDirectory(outputDir()); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java index 95b93cf..b118bd8 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java @@ -156,6 +156,7 @@ public class ScalarAssignmentTest extends AutomatedTestBase */ public void runScalarAssignmentTest( ControlFlowConstruct cfc, ValueType vt ) { + setOutputBuffering(true); String TEST_NAME = null; switch( cfc ) { @@ -188,5 +189,6 @@ public class ScalarAssignmentTest extends AutomatedTestBase boolean exceptionExpected = (cfc==ControlFlowConstruct.PARFOR)? true : false; //dependency analysis int expectedNumberOfJobs = -1; runTest(true, exceptionExpected, LanguageException.class, expectedNumberOfJobs); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java index a5bbd98..fb22b62 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java @@ -160,6 +160,7 @@ public class ZeroRowsColsMatrixTest extends AutomatedTestBase private void runEmptyMatrixTest( String testname, boolean rewrites, boolean emptyRet, ExecType et ) { + setOutputBuffering(true); ExecMode platformOld = rtplatform; switch( et ){ case SPARK: rtplatform = ExecMode.SPARK; break; @@ -205,5 +206,6 @@ public class ZeroRowsColsMatrixTest extends AutomatedTestBase rtplatform = platformOld; DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; } + setOutputBuffering(false); } } diff --git a/src/test/scripts/functions/misc/IPANnzPropagation1.dml b/src/test/scripts/functions/misc/IPANnzPropagation1.dml index 711302a..3884c08 100644 --- a/src/test/scripts/functions/misc/IPANnzPropagation1.dml +++ b/src/test/scripts/functions/misc/IPANnzPropagation1.dml @@ -24,7 +24,7 @@ foo = function(matrix[double] X) return (double sum) { sum = sum(X); } -X = rand(rows=1000, cols=1000000000, sparsity=1e-6) +X = rand(rows=1000, cols=100000000, sparsity=1e-6) s1 = foo(X); s2 = foo(X); print(s1+" "+s2); diff --git a/src/test/scripts/functions/misc/IPANnzPropagation2.dml b/src/test/scripts/functions/misc/IPANnzPropagation2.dml index de0a1df..fc4b213 100644 --- a/src/test/scripts/functions/misc/IPANnzPropagation2.dml +++ b/src/test/scripts/functions/misc/IPANnzPropagation2.dml @@ -24,8 +24,8 @@ foo = function(matrix[double] X) return (double sum) { sum = sum(X); } -X = rand(rows=1000, cols=1000000000, sparsity=1e-6) +X = rand(rows=1000, cols=100000000, sparsity=1e-6) s1 = foo(X); -X = rand(rows=1000, cols=1000000000, sparsity=1e-7) +X = rand(rows=1000, cols=100000000, sparsity=1e-7) s2 = foo(X); print(s1+" "+s2);
