[systemds] branch master updated: [MINOR] Fix FederatedTokenizeTest that was reading CSV files without headers

[baunsgaard]

This is an automated email from the ASF dual-hosted git repository.

baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new e1f4a6b  [MINOR] Fix FederatedTokenizeTest that was reading CSV files 
without headers
e1f4a6b is described below

commit e1f4a6b3c8fa7462581a9918b354fa169545ac37
Author: baunsgaard <baunsga...@tugraz.at>
AuthorDate: Mon Aug 23 13:16:33 2021 +0200

    [MINOR] Fix FederatedTokenizeTest that was reading CSV files without headers
    
    Remove unused import in test Utils
---
 src/test/java/org/apache/sysds/test/TestUtils.java               | 1 -
 .../functions/federated/primitives/FederatedTokenizeTest.java    | 9 +++++++--
 src/test/scripts/functions/federated/FederatedTokenizeTest.dml   | 4 ++--
 .../functions/federated/FederatedTokenizeTestReference.dml       | 3 +--
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java 
b/src/test/java/org/apache/sysds/test/TestUtils.java
index 9282d93..bd96a3f 100644
--- a/src/test/java/org/apache/sysds/test/TestUtils.java
+++ b/src/test/java/org/apache/sysds/test/TestUtils.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.SequenceFile.Writer;
 import org.apache.sysds.common.Types.FileFormat;
 import org.apache.sysds.common.Types.ValueType;
-import org.apache.sysds.runtime.DMLRuntimeException;
 import org.apache.sysds.runtime.data.TensorBlock;
 import org.apache.sysds.runtime.io.FrameWriter;
 import org.apache.sysds.runtime.io.FrameWriterFactory;
diff --git 
a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedTokenizeTest.java
 
b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedTokenizeTest.java
index e7c408d..5af8cb6 100644
--- 
a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedTokenizeTest.java
+++ 
b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedTokenizeTest.java
@@ -125,7 +125,7 @@ public class FederatedTokenizeTest extends 
AutomatedTestBase {
 
                // Run reference dml script with normal matrix
                fullDMLScriptName = HOME + TEST_NAME + "Reference.dml";
-               programArgs = new String[] {"-explain", "-args", DATASET_DIR + 
DATASET, HOME + TEST_NAME + ".json", expected("S")};
+               programArgs = new String[] {"-explain", "-args", input("AH"), 
HOME + TEST_NAME + ".json", expected("S")};
                runTest(null);
                // Run actual dml script with federated matrix
 
@@ -137,7 +137,12 @@ public class FederatedTokenizeTest extends 
AutomatedTestBase {
                        "in_S=" + input(HOME + TEST_NAME + ".json"), "rows=" + 
rows, "cols=" + cols,
                        "out_R=" + output("S")};
                runTest(null);
-
+               try{
+                       Thread.sleep(10000);
+               }
+               catch(Exception e){
+                       throw new RuntimeException(e);
+               }
                compareResults(1e-9);
                Assert.assertTrue(heavyHittersContainsString("fed_tokenize"));
                TestUtils.shutdownThreads(t1, t2, t3, t4);
diff --git a/src/test/scripts/functions/federated/FederatedTokenizeTest.dml 
b/src/test/scripts/functions/federated/FederatedTokenizeTest.dml
index e293306..ee24a2e 100644
--- a/src/test/scripts/functions/federated/FederatedTokenizeTest.dml
+++ b/src/test/scripts/functions/federated/FederatedTokenizeTest.dml
@@ -20,8 +20,8 @@
 #-------------------------------------------------------------
 
 F1 = federated(type="frame", addresses=list($in_X1, $in_X2, $in_X3),
-  ranges=list(list(0, 0), list(2, $cols), list(2, 0), list(4, $cols),
-  list(4, 0), list(6, $cols)));
+  ranges=list(list(0, 0), list(3, $cols), list(3, 0), list(6, $cols),
+  list(6, 0), list(9, $cols)));
 
 max_token = 2000;
 
diff --git 
a/src/test/scripts/functions/federated/FederatedTokenizeTestReference.dml 
b/src/test/scripts/functions/federated/FederatedTokenizeTestReference.dml
index f6dae76..5522955 100644
--- a/src/test/scripts/functions/federated/FederatedTokenizeTestReference.dml
+++ b/src/test/scripts/functions/federated/FederatedTokenizeTestReference.dml
@@ -19,8 +19,7 @@
 #
 #-------------------------------------------------------------
 
-F = read($1, data_type="frame", format="csv", sep=",");
-F = F[2:3, 1:4];
+F = read($1);
 F = rbind(F, rbind(F, F));
 
 max_token = 2000;