[systemds] branch main updated: [SYSTEMS-3432] Estimate NNZ/sparsity in bindResponses

[baunsgaard]

This is an automated email from the ASF dual-hosted git repository.

baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/main by this push:
     new 35f4e79b54 [SYSTEMS-3432] Estimate NNZ/sparsity in bindResponses
35f4e79b54 is described below

commit 35f4e79b5483415c568dbf93aeec9311e86fe87c
Author: Andreas Botzner <andr...@botzner.com>
AuthorDate: Mon Aug 29 09:46:22 2022 +0200

    [SYSTEMS-3432] Estimate NNZ/sparsity in bindResponses
    
    This commit analyze if the result matrix should be sparse before
    allocation when getting a federated response back to the controller,
    And allocate a sparse or dense matrix accordingly.
    
    Closes #1692
---
 .../runtime/controlprogram/federated/FederationUtils.java      | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git 
a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java
 
b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java
index 269d91e9c5..02aefb928c 100644
--- 
a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java
+++ 
b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java
@@ -145,7 +145,7 @@ public class FederationUtils {
                                        Lop.OPERAND_DELIMITOR + 
varOldOut.getName() + Lop.DATATYPE_PREFIX,
                                        Lop.OPERAND_DELIMITOR + 
String.valueOf(outputId) + Lop.DATATYPE_PREFIX);
                        }
-                       
+
                        fr[j] = new FederatedRequest(RequestType.EXEC_INST, 
outputId, (Object) linst[j]);
                }
                return fr;
@@ -539,7 +539,13 @@ public class FederationUtils {
        public static MatrixBlock bindResponses(List<Pair<FederatedRange, 
Future<FederatedResponse>>> readResponses, long[] dims)
                throws Exception
        {
-               MatrixBlock ret = new MatrixBlock((int) dims[0], (int) dims[1], 
false);
+               long totalNNZ = 0;
+               for(Pair<FederatedRange, Future<FederatedResponse>> 
readResponse : readResponses) {
+                       FederatedResponse response = 
readResponse.getRight().get();
+                       MatrixBlock multRes = (MatrixBlock) 
response.getData()[0];
+                       totalNNZ += multRes.getNonZeros();
+               }
+               MatrixBlock ret = new MatrixBlock((int) dims[0], (int) dims[1], 
MatrixBlock.evalSparseFormatInMemory(dims[0], dims[1], totalNNZ));
                for(Pair<FederatedRange, Future<FederatedResponse>> 
readResponse : readResponses) {
                        FederatedRange range = readResponse.getLeft();
                        FederatedResponse response = 
readResponse.getRight().get();