[systemds] branch master updated: [SYSTEMDS-3062] Fix runtime size propagation spark sample

[mboehm7]

This is an automated email from the ASF dual-hosted git repository.

mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new c7a5de0  [SYSTEMDS-3062] Fix runtime size propagation spark sample
c7a5de0 is described below

commit c7a5de08ff72f700533815ed453831355d98e77a
Author: Matthias Boehm <mboe...@gmail.com>
AuthorDate: Fri Jul 16 15:31:59 2021 +0200

    [SYSTEMDS-3062] Fix runtime size propagation spark sample
    
    This patch fixes a missing propagation of the number of rows on spark
    sample operations in case the compiler could not infer them, which led
    to exceptions if following operations need this info but are forced to
    spark operations as well.
---
 .../sysds/runtime/instructions/spark/RandSPInstruction.java       | 3 ++-
 .../sysds/test/functions/builtin/BuiltinMatrixProfileTest.java    | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git 
a/src/main/java/org/apache/sysds/runtime/instructions/spark/RandSPInstruction.java
 
b/src/main/java/org/apache/sysds/runtime/instructions/spark/RandSPInstruction.java
index 91511d4..421b365 100644
--- 
a/src/main/java/org/apache/sysds/runtime/instructions/spark/RandSPInstruction.java
+++ 
b/src/main/java/org/apache/sysds/runtime/instructions/spark/RandSPInstruction.java
@@ -756,7 +756,8 @@ public class RandSPInstruction extends UnarySPInstruction {
                JavaPairRDD<MatrixIndexes, MatrixBlock> mbRDD = 
                                
RDDConverterUtils.binaryCellToBinaryBlock(sec.getSparkContext(), miRDD, mcOut, 
true);
                
-               sec.getDataCharacteristics(output.getName()).setNonZeros(lrows);
+               //step 5: output handling, incl meta data
+               sec.getDataCharacteristics(output.getName()).set(mcOut);
                sec.setRDDHandleForVariable(output.getName(), mbRDD);
        }
        
diff --git 
a/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMatrixProfileTest.java
 
b/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMatrixProfileTest.java
index f0c227f..8664ab8 100644
--- 
a/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMatrixProfileTest.java
+++ 
b/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMatrixProfileTest.java
@@ -61,10 +61,10 @@ public class BuiltinMatrixProfileTest extends 
AutomatedTestBase
                runMatrixProfileTest(4, 0.6, "TRUE", ExecType.CP);
        }
 
-//     @Test
-//     public void testMatrixProfileSPARK() throws IOException {
-//             runMatrixProfileTest(4, 1.0, "FALSE", ExecType.SPARK);
-//     }
+       @Test
+       public void testMatrixProfileSPARK() throws IOException {
+               runMatrixProfileTest(4, 0.02, "FALSE", ExecType.SPARK);
+       }
 
        
        private void runMatrixProfileTest(Integer window_size, Double 
sample_percent, String is_verbose, ExecType instType) throws IOException