This is an automated email from the ASF dual-hosted git repository. mboehm7 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 98e0147711c543c6514a58ed28905a08db772e3c Author: Matthias Boehm <[email protected]> AuthorDate: Tue Jul 25 20:42:56 2023 +0200 [MINOR] Fix federated sparsity propagation test (correct input nnz) After some time debugging (the long chain of operations), it turned out that the recent fix of hop-level sparsity propagation on already solved all issues of #1863. However, the test wrote meta data files with an incorrect number of non-zeros which lead to incorrect nnz propagation in local (which serves as the reference) and thus mismatches with federated where the meta-data files are handled differently. --- .../sysds/runtime/iogen/CustomProperties.java | 3 ++- .../io/FederatedSparsityPropagationTest.java | 9 +++---- .../io/FederatedSparsityPropagationTest.dml | 12 ++++----- .../FederatedSparsityPropagationTestReference.dml | 31 ++++++++++------------ 4 files changed, 24 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/iogen/CustomProperties.java b/src/main/java/org/apache/sysds/runtime/iogen/CustomProperties.java index 8654f6f522..eeeaf5028c 100644 --- a/src/main/java/org/apache/sysds/runtime/iogen/CustomProperties.java +++ b/src/main/java/org/apache/sysds/runtime/iogen/CustomProperties.java @@ -26,7 +26,8 @@ import java.util.ArrayList; import java.util.HashSet; public class CustomProperties extends FileFormatProperties implements Serializable { - + private static final long serialVersionUID = -2877155313260718134L; + private MappingProperties mappingProperties; private RowIndexStructure rowIndexStructure; private ColIndexStructure colIndexStructure; diff --git a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedSparsityPropagationTest.java b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedSparsityPropagationTest.java index ef61cb7904..42165d6b68 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedSparsityPropagationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedSparsityPropagationTest.java @@ -38,7 +38,6 @@ import org.apache.sysds.runtime.controlprogram.context.ExecutionContextFactory; import org.apache.sysds.runtime.controlprogram.Program; import org.apache.sysds.runtime.controlprogram.ProgramBlock; import org.apache.sysds.common.Types.ExecMode; -import org.apache.sysds.runtime.meta.MatrixCharacteristics; import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; @@ -56,7 +55,6 @@ public class FederatedSparsityPropagationTest extends AutomatedTestBase { private final static int NUM_MATRICES = 15; private final static String TEST_CLASS_DIR = TEST_DIR + FederatedSparsityPropagationTest.class.getSimpleName() + "/"; - private final static int blocksize = 1024; @Parameterized.Parameter() public int rows; @Parameterized.Parameter(1) @@ -95,11 +93,10 @@ public class FederatedSparsityPropagationTest extends AutomatedTestBase { int fed_rows = rows / 2; int fed_cols = cols; - MatrixCharacteristics mc = new MatrixCharacteristics(fed_rows, fed_cols, blocksize, fed_rows * fed_cols); double[][] X1 = getRandomMatrix(fed_rows, fed_cols, 1, 3, sparsity, 3); double[][] X2 = getRandomMatrix(fed_rows, fed_cols, 1, 3, sparsity, 7); - writeInputMatrixWithMTD("X1", X1, false, mc); - writeInputMatrixWithMTD("X2", X2, false, mc); + writeInputMatrixWithMTD("X1", X1, false); + writeInputMatrixWithMTD("X2", X2, false); // empty script name because we don't execute any script, just start the worker fullDMLScriptName = ""; @@ -163,7 +160,7 @@ public class FederatedSparsityPropagationTest extends AutomatedTestBase { ArrayList<ProgramBlock> progBlocks = rtprog.getProgramBlocks(); ExecutionContext ec = ExecutionContextFactory.createContext(rtprog); - + // execute the first program block and obtain the nnz from the federation maps progBlocks.get(0).execute(ec); Map<String, Long> fedNNZ = getFedNNZ(ec); diff --git a/src/test/scripts/functions/federated/io/FederatedSparsityPropagationTest.dml b/src/test/scripts/functions/federated/io/FederatedSparsityPropagationTest.dml index 515f5d564b..87bc40a038 100644 --- a/src/test/scripts/functions/federated/io/FederatedSparsityPropagationTest.dml +++ b/src/test/scripts/functions/federated/io/FederatedSparsityPropagationTest.dml @@ -76,16 +76,14 @@ M14 = rev(X); # reshape M15 = matrix(X, rows=1, cols=(ncol(X)*nrow(X)), byrow=FALSE); -# FIXME: wrong nnz from cumsum instruction -# # cumulative sum -# M16 = cumsum(X); +# cumulative sum +M16 = cumsum(X); -# FIXME: wrong nnz from ternary instruction -# # ternary ifelse -# M17 = ifelse(X > X2, X, X2); +# ternary ifelse +M17 = ifelse(X > X2, X, X2); while(FALSE) {} Z = sum(M1) + sum(M2) + sum(M3) + sum(M4) + sum(M5) + sum(M6) + sum(M7) + sum(M8) + sum(M9) + sum(M10) + sum(M11) + sum(M12) + sum(M13) + sum(M14) + sum(M15) - /*+ sum(M16) + sum(M17)*/; + + sum(M16) + sum(M17); # NOTE: when adding tests, please remember to increment the number of matrices in the java test class diff --git a/src/test/scripts/functions/federated/io/FederatedSparsityPropagationTestReference.dml b/src/test/scripts/functions/federated/io/FederatedSparsityPropagationTestReference.dml index aca684dce7..9540351aa7 100644 --- a/src/test/scripts/functions/federated/io/FederatedSparsityPropagationTestReference.dml +++ b/src/test/scripts/functions/federated/io/FederatedSparsityPropagationTestReference.dml @@ -71,13 +71,11 @@ M14 = rev(X); # reshape M15 = matrix(X, rows=1, cols=(ncol(X)*nrow(X)), byrow=FALSE); -# FIXME: wrong nnz from cumsum instruction -# # cumulative sum -# M16 = cumsum(X); +# cumulative sum +M16 = cumsum(X); -# FIXME: wrong nnz from ternary instruction -# # ternary ifelse -# M17 = ifelse(X > X2, X, X2); +# ternary ifelse +M17 = ifelse(X > X2, X, X2); while(FALSE) { } @@ -93,13 +91,11 @@ NNZ_M5 = sum(M5 != 0); write(NNZ_M5, $out_Dir + "NNZ_M5"); NNZ_M6 = sum(M6 != 0); write(NNZ_M6, $out_Dir + "NNZ_M6"); -# FIXME: the not equal operation returns incorrect results -# DMLTranslator.rewriteHopsDAG rewrites the instruction sum(X != s) and introduces incorrect results -NNZ_M7 = (nrow(M7) * ncol(M7)) - sum(M7 == 0); +NNZ_M7 = sum(M7 != 0); write(NNZ_M7, $out_Dir + "NNZ_M7"); NNZ_M8 = sum(M8 != 0); write(NNZ_M8, $out_Dir + "NNZ_M8"); -NNZ_M9 = (nrow(M9) * ncol(M9)) - sum(M9 == 0); # FIXME: the not equal operation returns incorrect results +NNZ_M9 = sum(M9 != 0); write(NNZ_M9, $out_Dir + "NNZ_M9"); NNZ_M10 = sum(M10 != 0); write(NNZ_M10, $out_Dir + "NNZ_M10"); @@ -107,14 +103,15 @@ NNZ_M11 = sum(M11 != 0); write(NNZ_M11, $out_Dir + "NNZ_M11"); NNZ_M12 = sum(M12 != 0); write(NNZ_M12, $out_Dir + "NNZ_M12"); -NNZ_M13 = (nrow(M13) * ncol(M13)) - sum(M13 == 0); # FIXME: the not equal operation returns incorrect results +NNZ_M13 = sum(M13 != 0); write(NNZ_M13, $out_Dir + "NNZ_M13"); -NNZ_M14 = (nrow(M14) * ncol(M14)) - sum(M14 == 0); # FIXME: the not equal operation returns incorrect results +NNZ_M14 = sum(M14 != 0); write(NNZ_M14, $out_Dir + "NNZ_M14"); -NNZ_M15 = (nrow(M15) * ncol(M15)) - sum(M15 == 0); # FIXME: the not equal operation returns incorrect results +NNZ_M15 = sum(M15 != 0); write(NNZ_M15, $out_Dir + "NNZ_M15"); -# NNZ_M16 = sum(M16 != 0); -# write(NNZ_M16, $out_Dir + "NNZ_M16"); -# NNZ_M17 = sum(M17 != 0); -# write(NNZ_M17, $out_Dir + "NNZ_M17"); +NNZ_M16 = sum(M16 != 0); +write(NNZ_M16, $out_Dir + "NNZ_M16"); +NNZ_M17 = sum(M17 != 0); +write(NNZ_M17, $out_Dir + "NNZ_M17"); # NOTE: when adding tests, please remember to increment the number of matrices in the java test class +
