This is an automated email from the ASF dual-hosted git repository. mboehm7 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/systemds.git
commit e96e1cbafc53e90f64c08899cccbc1f02a9e46b3 Author: Matthias Boehm <[email protected]> AuthorDate: Sat Oct 31 21:41:51 2020 +0100 [MINOR] Fix null-pointer exceptions in new federated I/O tests In order to allow running these tests in isolation they need to enable output buffering and cannot rely on running in the same JVM where some other test might have already enabled it. --- .../federated/io/FederatedReaderTest.java | 31 +++++++--------------- .../federated/io/FederatedWriterTest.java | 26 +++++++----------- 2 files changed, 19 insertions(+), 38 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedReaderTest.java b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedReaderTest.java index c14ac1d..8fa2063 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedReaderTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedReaderTest.java @@ -21,7 +21,6 @@ package org.apache.sysds.test.functions.federated.io; import java.util.Arrays; import java.util.Collection; -import org.apache.sysds.api.DMLScript; import org.apache.sysds.common.Types; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; import org.apache.sysds.runtime.meta.MatrixCharacteristics; @@ -70,14 +69,10 @@ public class FederatedReaderTest extends AutomatedTestBase { } public void federatedRead(Types.ExecMode execMode) { - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - Types.ExecMode platformOld = rtplatform; - rtplatform = execMode; - if(rtplatform == Types.ExecMode.SPARK) { - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - } + Types.ExecMode oldPlatform = setExecMode(execMode); getAndLoadTestConfiguration(TEST_NAME); - + setOutputBuffering(true); + // write input matrices int halfRows = rows / 2; long[][] begins = new long[][] {new long[] {0, 0}, new long[] {halfRows, 0}}; @@ -95,15 +90,9 @@ public class FederatedReaderTest extends AutomatedTestBase { Thread t2 = startLocalFedWorkerThread(port2); String host = "localhost"; - MatrixObject fed = FederatedTestObjectConstructor.constructFederatedInput(rows, - cols, - blocksize, - host, - begins, - ends, - new int[] {port1, port2}, - new String[] {input("X1"), input("X2")}, - input("X.json")); + MatrixObject fed = FederatedTestObjectConstructor.constructFederatedInput( + rows, cols, blocksize, host, begins, ends, new int[] {port1, port2}, + new String[] {input("X1"), input("X2")}, input("X.json")); writeInputFederatedWithMTD("X.json", fed, null); try { @@ -120,16 +109,16 @@ public class FederatedReaderTest extends AutomatedTestBase { Assert.assertTrue(heavyHittersContainsString("fed_uak+")); // Verify output Assert.assertEquals(Double.parseDouble(refOut.split("\n")[0]), - Double.parseDouble(out.split("\n")[0]), - 0.00001); + Double.parseDouble(out.split("\n")[0]), 0.00001); } catch(Exception e) { e.printStackTrace(); Assert.assertTrue(false); } + finally { + resetExecMode(oldPlatform); + } TestUtils.shutdownThreads(t1, t2); - rtplatform = platformOld; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; } } diff --git a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedWriterTest.java b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedWriterTest.java index e03474d..587da82 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedWriterTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/io/FederatedWriterTest.java @@ -21,8 +21,7 @@ package org.apache.sysds.test.functions.federated.io; import java.util.Arrays; import java.util.Collection; -import org.apache.sysds.api.DMLScript; -import org.apache.sysds.common.Types; +import org.apache.sysds.common.Types.ExecMode; import org.apache.sysds.runtime.meta.MatrixCharacteristics; import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; @@ -65,17 +64,13 @@ public class FederatedWriterTest extends AutomatedTestBase { @Test public void federatedSinglenodeWrite() { - federatedWrite(Types.ExecMode.SINGLE_NODE); + federatedWrite(ExecMode.SINGLE_NODE); } - public void federatedWrite(Types.ExecMode execMode) { - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - Types.ExecMode platformOld = rtplatform; - rtplatform = execMode; - if(rtplatform == Types.ExecMode.SPARK) { - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - } + public void federatedWrite(ExecMode execMode) { + ExecMode oldPlatform = setExecMode(execMode); getAndLoadTestConfiguration(TEST_NAME); + setOutputBuffering(true); // write input matrices int halfRows = rows / 2; @@ -97,10 +92,7 @@ public class FederatedWriterTest extends AutomatedTestBase { fullDMLScriptName = SCRIPT_DIR + "functions/federated/io/FederatedReaderTestCreate.dml"; programArgs = new String[] {"-stats", "-explain", "-args", input("X1"), input("X2"), port1 + "", port2 + "", input("X.json")}; - // String writer = runTest(null).toString(); runTest(null); - // LOG.error(writer); - // LOG.error("Writing Done"); // Run reference dml script with normal matrix fullDMLScriptName = SCRIPT_DIR + "functions/federated/io/FederatedReaderTest.dml"; @@ -120,16 +112,16 @@ public class FederatedWriterTest extends AutomatedTestBase { // Verify output Assert.assertEquals(Double.parseDouble(refOut.split("\n")[0]), - Double.parseDouble(out.split("\n")[0]), - 0.00001); + Double.parseDouble(out.split("\n")[0]), 0.00001); } catch(Exception e) { e.printStackTrace(); Assert.assertTrue(false); } + finally { + resetExecMode(oldPlatform); + } TestUtils.shutdownThreads(t1, t2); - rtplatform = platformOld; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; } }
