(systemds) branch main updated: [SYSTEMDS-3846] Perftest: Fix parfor forced CP compilation

[mboehm7]

This is an automated email from the ASF dual-hosted git repository.

mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/main by this push:
     new 4d6c6b4cb7 [SYSTEMDS-3846] Perftest: Fix parfor forced CP compilation
4d6c6b4cb7 is described below

commit 4d6c6b4cb71c4ea1dc818739e601ab6a735c55ac
Author: Matthias Boehm <mboe...@gmail.com>
AuthorDate: Sun Apr 6 10:20:24 2025 +0200

    [SYSTEMDS-3846] Perftest: Fix parfor forced CP compilation
    
    The perftest runMSVM_10k_1k_dense_k5 reproducible failed on serializing
    the parfor body program for a remote spark parfor job, because there
    were remaining spark instructions (checkpoints). The reason was because
    the forced CP compilation before such remote jobs was only applied in
    a subset of cases and special hop properties were not correctly cleaned
    up. This patch fixes the general issue.
---
 src/main/java/org/apache/sysds/hops/recompile/Recompiler.java       | 6 +++++-
 .../org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java | 3 +--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/apache/sysds/hops/recompile/Recompiler.java 
b/src/main/java/org/apache/sysds/hops/recompile/Recompiler.java
index 63d0f1eab6..0aef8a1583 100644
--- a/src/main/java/org/apache/sysds/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysds/hops/recompile/Recompiler.java
@@ -1509,8 +1509,12 @@ public class Recompiler {
        public static void rSetExecType( Hop hop, ExecType etype ) {
                if( hop.isVisited() )
                        return;
-               //update function names
+               //update hop properties (exec type, spark lops)
                hop.setForcedExecType(etype);
+               if(etype == ExecType.CP) {
+                       hop.setRequiresCheckpoint(false);
+                       hop.setRequiresReblock(false);
+               }
                if( hop.getInput() != null )
                        for( Hop c : hop.getInput() )
                                rSetExecType(c, etype);
diff --git 
a/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java 
b/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
index 4c492131d9..85f41ad7fe 100644
--- 
a/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
+++ 
b/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
@@ -834,8 +834,7 @@ public class ParForProgramBlock extends ForProgramBlock {
        {
                // Step 0) check and compile to CP (if forced remote parfor)
                boolean flagForced = false;
-               if( FORCE_CP_ON_REMOTE_SPARK && (_optMode == POptMode.NONE 
-                       || (_optMode == POptMode.CONSTRAINED && 
_execMode==PExecMode.REMOTE_SPARK)) ) {
+               if( FORCE_CP_ON_REMOTE_SPARK && 
_execMode==PExecMode.REMOTE_SPARK ) {
                        //tid = 0  because replaced in remote parworker
                        flagForced = checkSparkAndRecompileToCP(0);
                }