[systemds] branch main updated: [SYSTEMDS-3471] Enable multi-threaded transformencode/apply

[arnabp20]

This is an automated email from the ASF dual-hosted git repository.

arnabp20 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/main by this push:
     new 220fd54aac [SYSTEMDS-3471] Enable multi-threaded transformencode/apply
220fd54aac is described below

commit 220fd54aace04f85764cdfbbcd8212ea2480d65f
Author: Arnab Phani <phaniar...@gmail.com>
AuthorDate: Wed Nov 30 16:46:16 2022 +0100

    [SYSTEMDS-3471] Enable multi-threaded transformencode/apply
    
    This patch changes the defaults of the Encode config flags
    to use the multi-threaded build, apply, allocation and
    compactions methods for CP transformencode/apply.
    
    Closes #1743
---
 src/main/java/org/apache/sysds/conf/DMLConfig.java                | 2 +-
 .../fed/MultiReturnParameterizedBuiltinFEDInstruction.java        | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java 
b/src/main/java/org/apache/sysds/conf/DMLConfig.java
index 937910d720..2225d89d89 100644
--- a/src/main/java/org/apache/sysds/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java
@@ -152,7 +152,7 @@ public class DMLConfig
                _defaultVals.put(CP_PARALLEL_IO,         "true" );
                _defaultVals.put(PARALLEL_TOKENIZE,      "false");
                _defaultVals.put(PARALLEL_TOKENIZE_NUM_BLOCKS, "64");
-               _defaultVals.put(PARALLEL_ENCODE,        "false" );
+               _defaultVals.put(PARALLEL_ENCODE,        "true" );
                _defaultVals.put(PARALLEL_ENCODE_STAGED, "false" );
                _defaultVals.put(PARALLEL_ENCODE_APPLY_BLOCKS, "-1");
                _defaultVals.put(PARALLEL_ENCODE_BUILD_BLOCKS, "-1");
diff --git 
a/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
 
b/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
index 5e6db1d792..2b14bd68d4 100644
--- 
a/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
+++ 
b/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
@@ -344,7 +344,9 @@ public class MultiReturnParameterizedBuiltinFEDInstruction 
extends ComputationFE
                                .createEncoder(_spec, colNames, 
fb.getNumColumns(), null, _offset, _offset + fb.getNumColumns());
 
                        // build necessary structures for encoding
-                       encoder.build(fb, 
OptimizerUtils.getTransformNumThreads()); // FIXME skip equi-height sorting
+                       //encoder.build(fb, 
OptimizerUtils.getTransformNumThreads()); // FIXME skip equi-height sorting
+                       // FIXME: Enabling multithreading intermittently hangs
+                       encoder.build(fb, 1);
                        fo.release();
 
                        // create federated response
@@ -375,7 +377,9 @@ public class MultiReturnParameterizedBuiltinFEDInstruction 
extends ComputationFE
                        // offset is applied on the Worker to shift the local 
encoders to their respective column
                        _encoder.applyColumnOffset();
                        // apply transformation
-                       MatrixBlock mbout = _encoder.apply(fb, 
OptimizerUtils.getTransformNumThreads());
+                       //MatrixBlock mbout = _encoder.apply(fb, 
OptimizerUtils.getTransformNumThreads());
+                       // FIXME: Enabling multithreading intermittently hangs
+                       MatrixBlock mbout = _encoder.apply(fb, 1);
 
                        // create output matrix object
                        MatrixObject mo = 
ExecutionContext.createMatrixObject(mbout);