This is an automated email from the ASF dual-hosted git repository. baunsgaard pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 8927a633e9dac73d94d617b6d9a9c7f0e26637ce Author: baunsgaard <[email protected]> AuthorDate: Sun Aug 29 16:58:54 2021 +0200 [MINOR] CLA no full decompress when writing to HDFS --- .../apache/sysds/runtime/controlprogram/caching/CacheableData.java | 4 ++-- .../org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java index 4034f76..d2b38a9 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java @@ -789,7 +789,6 @@ public abstract class CacheableData<T extends CacheBlock> extends Data */ public synchronized void exportData( int replication ) { exportData(_hdfsFileName, null, replication, null); - _hdfsFileExists = true; } public synchronized void exportData(String fName, String outputFormat) { @@ -945,7 +944,8 @@ public abstract class CacheableData<T extends CacheBlock> extends Data //CASE 4: data already in hdfs (do nothing, no need for export) LOG.trace(this.getDebugName() + ": Skip export to hdfs since data already exists."); } - + + _hdfsFileExists = true; if( DMLScript.STATISTICS ){ long t1 = System.nanoTime(); CacheStatistics.incrementExportTime(t1-t0); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java index 5a636ff..3194aa8 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java @@ -34,7 +34,6 @@ import org.apache.sysds.conf.ConfigurationManager; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.lops.Lop; import org.apache.sysds.runtime.DMLRuntimeException; -import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat; import org.apache.sysds.runtime.controlprogram.context.SparkExecutionContext; import org.apache.sysds.runtime.controlprogram.federated.FederatedRange; @@ -545,9 +544,6 @@ public class MatrixObject extends CacheableData<MatrixBlock> { ReaderWriterFederated.write(fname, this._fedMapping); } else if(_data != null) { - if(_data instanceof CompressedMatrixBlock) - _data = CompressedMatrixBlock.getUncompressed(_data); - MetaDataFormat iimd = (MetaDataFormat) _metaData; // Get the dimension information from the metadata stored within MatrixObject DataCharacteristics mc = iimd.getDataCharacteristics();
