[systemds] branch master updated: [MINOR] Check if NNZ is in mdt before parsing

[baunsgaard]

This is an automated email from the ASF dual-hosted git repository.

baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new 9c0d0b7  [MINOR] Check if NNZ is in mdt before parsing
9c0d0b7 is described below

commit 9c0d0b731be06f3112661a9c04740b91a3870e7e
Author: baunsgaard <baunsga...@tugraz.at>
AuthorDate: Wed Aug 26 16:43:02 2020 +0200

    [MINOR] Check if NNZ is in mdt before parsing
    
    This commit change the behavior of a federated worker if the
    metadata file does not contain the number of non zeros.
    It makes it default to parse into a dense representation when the input
    is small, and sparse representation if large.
---
 .../controlprogram/federated/FederatedWorkerHandler.java    | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git 
a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
 
b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
index fb8c494..5dbccb4 100644
--- 
a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
+++ 
b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
@@ -195,7 +195,18 @@ public class FederatedWorkerHandler extends 
ChannelInboundHandlerAdapter {
                                                        new 
FederatedWorkerHandlerException("Could not parse metadata file"));
                                        
mc.setRows(mtd.getLong(DataExpression.READROWPARAM));
                                        
mc.setCols(mtd.getLong(DataExpression.READCOLPARAM));
-                                       
mc.setNonZeros(mtd.getLong(DataExpression.READNNZPARAM));
+                                       
if(mtd.containsKey(DataExpression.READNNZPARAM)){
+                                               
mc.setNonZeros(mtd.getLong(DataExpression.READNNZPARAM));
+                                       }
+                                       else if (mc.getCols() * mc.getRows() < 
8000000){
+                                               // force dense allocation.
+                                               mc.setNonZeros(mc.getCols() 
*mc.getRows());
+                                       }
+                                       else{
+                                               // force sparse allocation 
+                                               
mc.setNonZeros((long)(mc.getCols() * mc.getRows() * 0.35));
+                                       }
+                                       
                                        cd = (CacheableData<?>) 
PrivacyPropagator.parseAndSetPrivacyConstraint(cd, mtd);
                                        fmt = 
FileFormat.safeValueOf(mtd.getString(DataExpression.FORMAT_TYPE));
                                }