This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 5b71a03736 [SYSTEMDS-3813] Fix incorrect file format on list write
5b71a03736 is described below
commit 5b71a03736f36ace6bd6af6512fb3a4b8198ac9e
Author: Matthias Boehm <[email protected]>
AuthorDate: Fri Jan 17 13:18:00 2025 +0100
[SYSTEMDS-3813] Fix incorrect file format on list write
The writing of lists always produced individual matrix/frame files in
text format (even though the metadata of the list indicated otherwise).
This patch now properly writes the individual files in the correct
format.
---
src/main/java/org/apache/sysds/lops/Data.java | 10 ++++------
.../sysds/runtime/instructions/cp/VariableCPInstruction.java | 3 ++-
.../org/apache/sysds/test/functions/io/ReadWriteListTest.java | 9 ++++++++-
3 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/src/main/java/org/apache/sysds/lops/Data.java
b/src/main/java/org/apache/sysds/lops/Data.java
index a0546904c0..d47b010e98 100644
--- a/src/main/java/org/apache/sysds/lops/Data.java
+++ b/src/main/java/org/apache/sysds/lops/Data.java
@@ -276,12 +276,10 @@ public class Data extends Lop
OutputParameters oparams = getOutputParameters();
if ( _op.isWrite() ) {
sb.append( OPERAND_DELIMITOR );
- FileFormat fmt = null;
- if ( getDataType() == DataType.MATRIX || getDataType()
== DataType.FRAME )
- fmt = oparams.getFormat();
- else // scalars will always be written in text format
- fmt = FileFormat.TEXT;
-
+ // scalars will always be written in text format
+ FileFormat fmt = getDataType().isScalar() ?
+ FileFormat.TEXT : oparams.getFormat();
+
//format literal or variable
Lop fmtLop =
_inputParams.get(DataExpression.FORMAT_TYPE);
String fmtLabel = (fmt!=FileFormat.UNKNOWN) ?
fmt.toString() : fmtLop.getOutputParameters().getLabel();
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
index 3ae0a96f0b..6ed292bbb0 100644
---
a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
+++
b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
@@ -1084,7 +1084,8 @@ public class VariableCPInstruction extends CPInstruction
implements LineageTrace
}
else if( getInput1().getDataType() == DataType.LIST ) {
ListObject lo = ec.getListObject(getInput1().getName());
- ListWriter.writeListToHDFS(lo, fname, fmtStr,
_formatProperties);
+ int blen = Integer.parseInt(getInput4().getName());
+ ListWriter.writeListToHDFS(lo, fname, fmtStr, new
FileFormatProperties(blen));
}
}
diff --git
a/src/test/java/org/apache/sysds/test/functions/io/ReadWriteListTest.java
b/src/test/java/org/apache/sysds/test/functions/io/ReadWriteListTest.java
index 7894c6da8b..f8fb7d9051 100644
--- a/src/test/java/org/apache/sysds/test/functions/io/ReadWriteListTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/io/ReadWriteListTest.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysds.common.Types.ExecMode;
import org.apache.sysds.common.Types.FileFormat;
+import org.apache.sysds.runtime.meta.MetaDataAll;
import org.apache.sysds.runtime.util.HDFSTool;
import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestConfiguration;
@@ -120,8 +121,14 @@ public class ReadWriteListTest extends AutomatedTestBase {
fullDMLScriptName = HOME + TEST_NAME2 + ".dml";
programArgs = new String[]{"-args", output("L"),
output("R2")};
runTest(true, false, null, -1);
- double val2 =
HDFSTool.readDoubleFromHDFSFile(output("R2"));
+ //check meta data, incl implicitly format
+ MetaDataAll meta = getMetaData("L", OUTPUT_DIR);
+ Assert.assertEquals(format.toString(),
meta.getFormatTypeString());
+ Assert.assertEquals(4, meta.getDim1());
+ Assert.assertEquals(1, meta.getDim2());
+
+ double val2 =
HDFSTool.readDoubleFromHDFSFile(output("R2"));
Assert.assertEquals(Double.valueOf(val1),
Double.valueOf(val2), eps);
}
catch(IOException e) {
