This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 6652d43939 [SYSTEMDS-3787] Fix missing transformencode update
federated backend
6652d43939 is described below
commit 6652d439393029b84830c792c9cb8da82e49f298
Author: Matthias Boehm <[email protected]>
AuthorDate: Thu Oct 24 12:48:33 2024 +0200
[SYSTEMDS-3787] Fix missing transformencode update federated backend
This patch adds support for the new transform encode w/o metadata
output also to the federated backend, and fixes the robustness for
handling empty frame blocks.
---
.../sysds/runtime/frame/data/FrameBlock.java | 7 ++++---
...ltiReturnParameterizedBuiltinCPInstruction.java | 4 ++++
...tiReturnParameterizedBuiltinFEDInstruction.java | 24 ++++++++++++++--------
...ltiReturnParameterizedBuiltinSPInstruction.java | 4 ++++
4 files changed, 28 insertions(+), 11 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
b/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
index 4896d82f28..61875d2e14 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
@@ -925,9 +925,10 @@ public class FrameBlock implements CacheBlock<FrameBlock>,
Externalizable {
public boolean isShallowSerialize(boolean inclConvert) {
// shallow serialize if non-string schema because a frame block
// is always dense but strings have large array overhead per
cell
- for(int j = 0; j < _schema.length; j++)
- if(!_coldata[j].isShallowSerialize())
- return false;
+ if( _schema != null )
+ for(int j = 0; j < _schema.length; j++)
+ if(!_coldata[j].isShallowSerialize())
+ return false;
return true;
}
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
index f5ba63b0ef..e5e486752d 100644
---
a/src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
+++
b/src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
@@ -123,4 +123,8 @@ public class MultiReturnParameterizedBuiltinCPInstruction
extends ComputationCPI
}
return ret;
}
+
+ public boolean getMetaReturn() {
+ return _metaReturn;
+ }
}
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
index 416c2838af..917226fef7 100644
---
a/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
+++
b/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
@@ -70,10 +70,12 @@ import org.apache.sysds.runtime.util.IndexRange;
public class MultiReturnParameterizedBuiltinFEDInstruction extends
ComputationFEDInstruction {
protected final List<CPOperand> _outputs;
-
+ protected final boolean _metaReturn;
+
private MultiReturnParameterizedBuiltinFEDInstruction(Operator op,
CPOperand input1, CPOperand input2,
- List<CPOperand> outputs, String opcode, String istr) {
+ List<CPOperand> outputs, boolean metaReturn, String opcode,
String istr) {
super(FEDType.MultiReturnParameterizedBuiltin, op, input1,
input2, null, opcode, istr);
+ _metaReturn = metaReturn;
_outputs = outputs;
}
@@ -104,13 +106,13 @@ public class
MultiReturnParameterizedBuiltinFEDInstruction extends ComputationFE
private static MultiReturnParameterizedBuiltinFEDInstruction
parseInstruction(
MultiReturnParameterizedBuiltinCPInstruction instr) {
return new
MultiReturnParameterizedBuiltinFEDInstruction(instr.getOperator(),
instr.input1, instr.input2,
- instr.getOutputs(), instr.getOpcode(),
instr.getInstructionString());
+ instr.getOutputs(), instr.getMetaReturn(),
instr.getOpcode(), instr.getInstructionString());
}
private static MultiReturnParameterizedBuiltinFEDInstruction
parseInstruction(
MultiReturnParameterizedBuiltinSPInstruction instr) {
return new
MultiReturnParameterizedBuiltinFEDInstruction(instr.getOperator(),
instr.input1, instr.input2,
- instr.getOutputs(), instr.getOpcode(),
instr.getInstructionString());
+ instr.getOutputs(), instr.getMetaReturn(),
instr.getOpcode(), instr.getInstructionString());
}
public static MultiReturnParameterizedBuiltinFEDInstruction
parseInstruction(String str) {
@@ -122,9 +124,15 @@ public class MultiReturnParameterizedBuiltinFEDInstruction
extends ComputationFE
// one input and two outputs
CPOperand in1 = new CPOperand(parts[1]);
CPOperand in2 = new CPOperand(parts[2]);
- outputs.add(new CPOperand(parts[3],
Types.ValueType.FP64, Types.DataType.MATRIX));
- outputs.add(new CPOperand(parts[4],
Types.ValueType.STRING, Types.DataType.FRAME));
- return new
MultiReturnParameterizedBuiltinFEDInstruction(null, in1, in2, outputs, opcode,
str);
+ int pos = 3;
+ boolean metaReturn = true;
+ System.out.println(Arrays.toString(parts));
+ if( parts.length == 7 ) //no need for meta data
+ metaReturn = new
CPOperand(parts[pos++]).getLiteral().getBooleanValue();
+ outputs.add(new CPOperand(parts[pos],
Types.ValueType.FP64, Types.DataType.MATRIX));
+ outputs.add(new CPOperand(parts[pos+1],
Types.ValueType.STRING, Types.DataType.FRAME));
+ return new
MultiReturnParameterizedBuiltinFEDInstruction(
+ null, in1, in2, outputs, metaReturn, opcode,
str);
}
else {
throw new DMLRuntimeException("Invalid opcode in
MultiReturnBuiltin instruction: " + opcode);
@@ -198,7 +206,7 @@ public class MultiReturnParameterizedBuiltinFEDInstruction
extends ComputationFE
encodeFederatedFrames(fedMapping, globalEncoder,
ec.getMatrixObject(getOutput(0)));
// release input and outputs
- ec.setFrameOutput(getOutput(1).getName(), meta);
+ ec.setFrameOutput(getOutput(1).getName(), _metaReturn ? meta :
new FrameBlock());
}
private class EncoderColnames {
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
index ebd1bf7b44..90683eab29 100644
---
a/src/main/java/org/apache/sysds/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
+++
b/src/main/java/org/apache/sysds/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
@@ -468,4 +468,8 @@ public class MultiReturnParameterizedBuiltinSPInstruction
extends ComputationSPI
return ret.iterator();
}
}
+
+ public boolean getMetaReturn() {
+ return _metaReturn;
+ }
}
