This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 4b1d150 [SYSTEMDS-3050] MatrixBlock Hardening
4b1d150 is described below
commit 4b1d150b900c506818532c519c61155530077852
Author: baunsgaard <[email protected]>
AuthorDate: Mon Aug 23 13:25:24 2021 +0200
[SYSTEMDS-3050] MatrixBlock Hardening
This commit change many of the methods defined in MatrixBlock to
final, This makes it easier to expand the MatrixBlock object in the
CompressedMatrixBlock class.
Future work is to incorporate the CompressedMatrixBlock class into the
MatrixBlock, and separate out more of the Sparse vs Dense logic from
MatrixBlock.
Minor additions:
- Left and right matrix multiplication optimizations
- CompressedMatrixBlock cleanup
- Additional end to end tests of compression
[MINOR] Remove sleep in Federated Tokenize Test
addd
---
.../runtime/compress/CompressedMatrixBlock.java | 720 +++++++++++++--------
.../runtime/compress/lib/CLALibLeftMultBy.java | 57 +-
.../runtime/compress/lib/CLALibRightMultBy.java | 13 +-
.../sysds/runtime/matrix/data/MatrixBlock.java | 223 ++++---
.../compress/configuration/CompressBase.java | 2 +-
.../compress/configuration/CompressForce.java | 10 +
.../primitives/FederatedTokenizeTest.java | 6 -
.../compress/SystemDS-config-compress.xml | 25 +
.../functions/compress/compress_ewbm_left.dml | 1 +
.../functions/compress/compress_ewbm_right.dml | 1 +
.../compress_01.dml} | 16 +-
.../compress_02.dml} | 17 +-
.../compress_pca.dml} | 16 +-
.../compress_scale.dml} | 16 +-
.../compress_seq.dml} | 17 +-
.../compress_submean.dml} | 19 +-
.../functions/compress/densifying/large.ijv | 2 +
.../functions/compress/densifying/large.ijv.mtd | 8 +
.../functions/compress/densifying/small.ijv | 2 +
.../functions/compress/densifying/small.ijv.mtd | 8 +
.../WorkloadAnalysisLmCG.dml} | 28 +-
.../WorkloadAnalysisLmDS.dml} | 28 +-
.../compress/workload/WorkloadAnalysisMLogReg.dml | 6 +-
...MLogReg.dml => WorkloadAnalysisSliceFinder.dml} | 26 +-
.../functions/federated/FederatedTokenizeTest.dml | 4 +-
.../federated/FederatedTokenizeTestReference.dml | 3 +-
26 files changed, 804 insertions(+), 470 deletions(-)
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
index 26fa535..bae7bae 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
@@ -27,6 +27,7 @@ import java.io.ObjectOutput;
import java.lang.ref.SoftReference;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -34,12 +35,11 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import org.apache.commons.lang.NotImplementedException;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.math3.random.Well1024a;
import org.apache.sysds.api.DMLScript;
+import org.apache.sysds.common.Types.CorrectionLocationType;
import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.conf.DMLConfig;
import org.apache.sysds.hops.OptimizerUtils;
@@ -48,8 +48,6 @@ import org.apache.sysds.lops.MapMultChain.ChainType;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.colgroup.AColGroup;
import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType;
-import org.apache.sysds.runtime.compress.colgroup.ColGroupCompressed;
-import org.apache.sysds.runtime.compress.colgroup.ColGroupConst;
import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
import org.apache.sysds.runtime.compress.colgroup.ColGroupIO;
import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed;
@@ -65,7 +63,9 @@ import org.apache.sysds.runtime.compress.lib.CLALibSquash;
import org.apache.sysds.runtime.controlprogram.caching.CacheBlock;
import org.apache.sysds.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
+import org.apache.sysds.runtime.data.SparseRow;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.functionobjects.Builtin.BuiltinCode;
import org.apache.sysds.runtime.functionobjects.KahanPlus;
@@ -78,6 +78,7 @@ import org.apache.sysds.runtime.instructions.cp.CM_COV_Object;
import org.apache.sysds.runtime.instructions.cp.ScalarObject;
import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue;
import org.apache.sysds.runtime.matrix.data.CTableMap;
+import org.apache.sysds.runtime.matrix.data.IJV;
import org.apache.sysds.runtime.matrix.data.LibMatrixBincell;
import org.apache.sysds.runtime.matrix.data.LibMatrixDatagen;
import org.apache.sysds.runtime.matrix.data.LibMatrixReorg;
@@ -100,22 +101,15 @@ import
org.apache.sysds.runtime.matrix.operators.TernaryOperator;
import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
import org.apache.sysds.runtime.util.CommonThreadPool;
import org.apache.sysds.runtime.util.IndexRange;
-import org.apache.sysds.runtime.util.SortUtils;
import org.apache.sysds.utils.DMLCompressionStatistics;
public class CompressedMatrixBlock extends MatrixBlock {
private static final Log LOG =
LogFactory.getLog(CompressedMatrixBlock.class.getName());
- private static final long serialVersionUID = 7319372019143154058L;
+ private static final long serialVersionUID = 73193720143154058L;
protected List<AColGroup> _colGroups;
/**
- * list of lengths of dictionaries, including a longest length in left
variable. Note should not be called directly
- * since it is constructed on first use, on calls to : getMaxNumValues()
- */
- protected Pair<Integer, int[]> v = null;
-
- /**
* Boolean specifying if the colGroups are overlapping each other. This
happens after a right matrix multiplication.
*/
protected boolean overlappingColGroups = false;
@@ -125,13 +119,10 @@ public class CompressedMatrixBlock extends MatrixBlock {
*/
protected SoftReference<MatrixBlock> decompressedVersion;
- /**
- * Constructor for building an empty Compressed Matrix block object.
- *
- * OBS! Only to be used for serialization.
- */
public CompressedMatrixBlock() {
- super();
+ super(true);
+ sparse = false;
+ nonZeros = -1;
}
/**
@@ -143,33 +134,47 @@ public class CompressedMatrixBlock extends MatrixBlock {
* @param cl number of columns
*/
public CompressedMatrixBlock(int rl, int cl) {
- super(rl, cl, true);
- sparseBlock = null;
- denseBlock = null;
+ super(true);
+ rlen = rl;
+ clen = cl;
+ sparse = false;
nonZeros = -1;
}
/**
- * "Copy" constructor to populate this compressed block with the
uncompressed metadata contents of a conventional
- * block. Does not compress the block.
+ * Copy constructor taking that CompressedMatrixBlock and populate this
new compressedMatrixBlock with pointers to
+ * the same columnGroups.
*
- * @param that matrix block
+ * @param that CompressedMatrixBlock to copy values from
*/
- protected CompressedMatrixBlock(MatrixBlock that) {
- super(that.getNumRows(), that.getNumColumns(), true);
- sparseBlock = null;
- denseBlock = null;
- nonZeros = that.getNonZeros();
- }
-
public CompressedMatrixBlock(CompressedMatrixBlock that) {
- super(that.getNumRows(), that.getNumColumns(), true);
+ super(true);
+ rlen = that.getNumRows();
+ clen = that.getNumColumns();
this.copyCompressedMatrix(that);
}
- public boolean isSingleUncompressedGroup() {
- return(_colGroups != null && _colGroups.size() == 1 &&
- _colGroups.get(0).getCompType() ==
CompressionType.UNCOMPRESSED);
+ /**
+ * Copy constructor taking an uncompressedMatrixBlock to copy metadata
from also while copying metadata, a soft
+ * reference is constructed to the uncompressed matrixBlock, to allow
quick decompressions if the program is not
+ * under memory pressure.
+ *
+ * This method is used in the CompressionFactory.
+ *
+ * @param uncompressedMatrixBlock An uncompressed Matrix to copy
metadata from.
+ */
+ protected CompressedMatrixBlock(MatrixBlock uncompressedMatrixBlock) {
+ super(true);
+ rlen = uncompressedMatrixBlock.getNumRows();
+ clen = uncompressedMatrixBlock.getNumColumns();
+ sparse = false;
+ nonZeros = uncompressedMatrixBlock.getNonZeros();
+ decompressedVersion = new
SoftReference<>(uncompressedMatrixBlock);
+ }
+
+ @Override
+ public void reset(int rl, int cl, boolean sp, long estnnz, double val) {
+ throw new DMLCompressionException("Invalid to reset a
Compressed MatrixBlock");
}
/**
@@ -184,107 +189,72 @@ public class CompressedMatrixBlock extends MatrixBlock {
_colGroups.add(cg);
}
+ /**
+ * Replace the column groups in this CompressedMatrixBlock with the
given column groups
+ *
+ * @param colGroups new ColGroups in the MatrixBlock
+ */
public void allocateColGroupList(List<AColGroup> colGroups) {
_colGroups = colGroups;
}
+ /**
+ * Get the column groups of this CompressedMatrixBlock
+ *
+ * @return the column groups
+ */
public List<AColGroup> getColGroups() {
return _colGroups;
}
/**
- * Decompress block.
+ * Decompress block into a MatrixBlock
*
* @return a new uncompressed matrix block containing the contents of
this block
*/
public MatrixBlock decompress() {
-
- Timing time = new Timing(true);
-
- if(decompressedVersion != null && decompressedVersion.get() !=
null){
- if(DMLScript.STATISTICS || LOG.isDebugEnabled()) {
- double t = time.stop();
- LOG.debug("decompressed block was in soft
reference.");
- DMLCompressionStatistics.addDecompressTime(t,
1);
- }
- return decompressedVersion.get();
- }
-
-
- long nnz = getNonZeros() == -1 ? recomputeNonZeros() : nonZeros;
- if(isEmpty())
- return new MatrixBlock(rlen, clen, true, 0);
-
- // preallocation sparse rows to avoid repeated reallocations
- MatrixBlock ret =
getUncompressedColGroupAndRemoveFromListOfColGroups();
- if(ret != null && getColGroups().size() == 0)
- return ret;
- else if(ret == null)
- ret = new MatrixBlock(rlen, clen, false, -1);
- ret.setNonZeros(nnz);
- ret.allocateDenseBlock();
- decompress(ret);
-
- ret.examSparsity();
- if(DMLScript.STATISTICS || LOG.isDebugEnabled()) {
- double t = time.stop();
- LOG.debug("decompressed block w/ k=" + 1 + " in " + t +
"ms.");
- DMLCompressionStatistics.addDecompressTime(t, 1);
- }
-
- decompressedVersion = new SoftReference<>(ret);
- return ret;
- }
-
- private MatrixBlock decompress(MatrixBlock ret) {
-
- for(AColGroup grp : _colGroups)
- grp.decompressToBlockUnSafe(ret, 0, rlen, 0);
-
- if(ret.isInSparseFormat())
- ret.sortSparseRows();
-
- if(this.isOverlapping()) {
- ret.recomputeNonZeros();
- ret.examSparsity();
- }
-
- return ret;
+ return decompress(1);
}
/**
- * Decompress block.
+ * Decompress block into a MatrixBlock
*
* @param k degree of parallelism
* @return a new uncompressed matrix block containing the contents of
this block
*/
public MatrixBlock decompress(int k) {
- if(k <= 1)
- return decompress();
+ // Early out if empty.
+ if(isEmpty())
+ return new MatrixBlock(rlen, clen, true, 0);
- Timing time = new Timing(true);
+ // Early out if decompressed version already is cached
+ MatrixBlock ret = getCachedDecompressed();
+ if(ret != null)
+ return ret;
- if(decompressedVersion != null && decompressedVersion.get() !=
null){
- if(DMLScript.STATISTICS || LOG.isDebugEnabled()) {
- double t = time.stop();
- LOG.debug("decompressed block was in soft
reference.");
- DMLCompressionStatistics.addDecompressTime(t,
k);
- }
- return decompressedVersion.get();
- }
+ Timing time = new Timing(true);
+ ret = getUncompressedColGroupAndRemoveFromListOfColGroups();
- MatrixBlock ret =
getUncompressedColGroupAndRemoveFromListOfColGroups();
if(ret != null && getColGroups().size() == 0)
- return ret;
+ return ret; // if uncompressedColGroup is only colGroup.
else if(ret == null)
ret = new MatrixBlock(rlen, clen, false, -1);
+
ret.allocateDenseBlock();
- decompress(ret, k);
+
+ if(k == 1)
+ decompress(ret);
+ else
+ decompress(ret, k);
+
+ if(this.isOverlapping())
+ ret.recomputeNonZeros();
ret.examSparsity();
+
if(DMLScript.STATISTICS || LOG.isDebugEnabled()) {
double t = time.stop();
- LOG.debug("decompressed block w/ k=" + k + " in " +
time.stop() + "ms.");
+ LOG.debug("decompressed block w/ k=" + k + " in " + t +
"ms.");
DMLCompressionStatistics.addDecompressTime(t, k);
}
@@ -292,32 +262,55 @@ public class CompressedMatrixBlock extends MatrixBlock {
return ret;
}
- public MatrixBlock decompress(MatrixBlock ret, int k) {
+ private MatrixBlock decompress(MatrixBlock ret) {
+
+ ret.setNonZeros(nonZeros == -1 && !this.isOverlapping() ?
recomputeNonZeros() : nonZeros);
+ final int block = (int) Math.ceil((double)
(CompressionSettings.BITMAP_BLOCK_SZ) / getNumColumns());
+ final int blklen = block > 1000 ? block + 1000 - block % 1000 :
Math.max(64, block);
+ for(int i = 0; i < getNumRows(); i += blklen)
+ for(AColGroup grp : _colGroups)
+ grp.decompressToBlockUnSafe(ret, i, Math.min(i
+ blklen, rlen));
+
+ return ret;
+ }
+
+ private MatrixBlock decompress(MatrixBlock ret, int k) {
try {
- ExecutorService pool = CommonThreadPool.get(k);
- int rlen = getNumRows();
- final int blkz = CompressionSettings.BITMAP_BLOCK_SZ;
- int blklen = (int) Math.max(64, Math.ceil((double)
(blkz) / getNumColumns()));
- ArrayList<DecompressTask> tasks = new ArrayList<>();
+ final ExecutorService pool = CommonThreadPool.get(k);
+ final int rlen = getNumRows();
+ final int block = (int) Math.ceil((double)
(CompressionSettings.BITMAP_BLOCK_SZ) / getNumColumns());
+ final int blklen = block > 1000 ? block + 1000 - block
% 1000 : Math.max(64, block);
+ final ArrayList<DecompressTask> tasks = new
ArrayList<>();
for(int i = 0; i * blklen < getNumRows(); i++)
tasks.add(new DecompressTask(_colGroups, ret, i
* blklen, Math.min((i + 1) * blklen, rlen),
overlappingColGroups));
List<Future<Long>> rtasks = pool.invokeAll(tasks);
pool.shutdown();
+
+ long nnz = 0;
for(Future<Long> rt : rtasks)
- rt.get();
+ nnz += rt.get();
+ ret.setNonZeros(nnz);
}
catch(InterruptedException | ExecutionException ex) {
throw new DMLCompressionException("Parallel
decompression failed", ex);
}
- if(this.isOverlapping()) {
- ret.recomputeNonZeros();
- ret.examSparsity();
- }
return ret;
}
+ private MatrixBlock getCachedDecompressed() {
+ if(decompressedVersion != null) {
+ final MatrixBlock mb = decompressedVersion.get();
+ if(mb != null) {
+
DMLCompressionStatistics.addDecompressCacheCount();
+ LOG.debug("Decompressed block was in soft
reference.");
+ return mb;
+ }
+ }
+ return null;
+ }
+
private MatrixBlock
getUncompressedColGroupAndRemoveFromListOfColGroups() {
// If we have a uncompressed column group that covers all of
the matrix,
// it makes sense to use as the decompression target.
@@ -364,7 +357,26 @@ public class CompressedMatrixBlock extends MatrixBlock {
}
return nonZeros;
+ }
+ @Override
+ public long recomputeNonZeros(int rl, int ru) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public long recomputeNonZeros(int rl, int ru, int cl, int cu) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public long getInMemorySize() {
+ return estimateCompressedSizeInMemory();
+ }
+
+ @Override
+ public long estimateSizeInMemory() {
+ return estimateCompressedSizeInMemory();
}
/**
@@ -384,9 +396,11 @@ public class CompressedMatrixBlock extends MatrixBlock {
public static long baseSizeInMemory() {
long total = 16; // Object header
- total += 40; // Matrix Block elements
+ total += getHeaderSize(); // Matrix Block elements
total += 8; // Col Group Ref
- total += 2 + 6; // Booleans plus padding
+ total += 8; // v reference
+ total += 8; // soft reference to decompressed version
+ total += 1 + 7; // Booleans plus padding
total += 40; // Col Group Array List
return total;
@@ -394,6 +408,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
@Override
public double quickGetValue(int r, int c) {
+ // throw new NotImplementedException("Should not call quick get
Value on Compressed Matrix Block");
if(isOverlapping()) {
double v = 0.0;
for(AColGroup group : _colGroups)
@@ -419,6 +434,11 @@ public class CompressedMatrixBlock extends MatrixBlock {
}
@Override
+ public long estimateSizeOnDisk() {
+ return getExactSizeOnDisk();
+ }
+
+ @Override
public void readFields(DataInput in) throws IOException {
// deserialize compressed block
rlen = in.readInt();
@@ -477,42 +497,35 @@ public class CompressedMatrixBlock extends MatrixBlock {
}
@Override
- public MatrixBlock append(MatrixBlock that, MatrixBlock ret) {
- return CLALibAppend.append(this, that);
- }
-
- @Override
- public MatrixBlock append(MatrixBlock that, MatrixBlock ret, boolean
cbind) {
- if(cbind) // use supported operation
- return append(that, ret);
- printDecompressWarning("append-rbind", that);
- MatrixBlock left = getUncompressed();
- MatrixBlock right = getUncompressed(that);
- return left.append(right, ret, cbind);
+ public MatrixBlock append(MatrixBlock[] that, MatrixBlock ret, boolean
cbind) {
+ if(cbind && that.length == 1) {
+ return CLALibAppend.append(this, that[0]);
+ }
+ else {
+ MatrixBlock left = getUncompressed("append list or
r-bind not supported in compressed");
+ MatrixBlock[] thatUC = new MatrixBlock[that.length];
+ for(int i = 0; i < that.length; i++)
+ thatUC[i] = getUncompressed(that[i]);
+ return left.append(thatUC, ret, cbind);
+ }
}
@Override
public void append(MatrixValue v2, ArrayList<IndexedMatrixValue>
outlist, int blen, boolean cbind, boolean m2IsLast,
int nextNCol) {
- printDecompressWarning("append", (MatrixBlock) v2);
- MatrixBlock left = getUncompressed();
+ MatrixBlock left = getUncompressed("append ArrayList");
MatrixBlock right = getUncompressed(v2);
left.append(right, outlist, blen, cbind, m2IsLast, nextNCol);
}
@Override
- public MatrixBlock chainMatrixMultOperations(MatrixBlock v, MatrixBlock
w, MatrixBlock out, ChainType ctype) {
- return chainMatrixMultOperations(v, w, out, ctype, 1);
- }
-
- @Override
public MatrixBlock chainMatrixMultOperations(MatrixBlock v, MatrixBlock
w, MatrixBlock out, ChainType ctype,
int k) {
checkMMChain(ctype, v, w);
-
// multi-threaded MMChain of single uncompressed ColGroup
- if(isSingleUncompressedGroup())
+ if(_colGroups != null && _colGroups.size() == 1 &&
+ _colGroups.get(0).getCompType() ==
CompressionType.UNCOMPRESSED)
return ((ColGroupUncompressed)
_colGroups.get(0)).getData().chainMatrixMultOperations(v, w, out, ctype, k);
// prepare result
@@ -526,7 +539,8 @@ public class CompressedMatrixBlock extends MatrixBlock {
return out;
BinaryOperator bop = new
BinaryOperator(Multiply.getMultiplyFnObject());
- boolean allowOverlap =
ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.COMPRESSED_OVERLAPPING);
+ boolean allowOverlap =
ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.COMPRESSED_OVERLAPPING)
&&
+ v.getNumColumns() > 1;
MatrixBlock tmp = CLALibRightMultBy.rightMultByMatrix(this, v,
null, k, allowOverlap);
if(ctype == ChainType.XtwXv) {
@@ -549,13 +563,9 @@ public class CompressedMatrixBlock extends MatrixBlock {
}
@Override
- public MatrixBlock aggregateBinaryOperations(MatrixBlock m1,
MatrixBlock m2, AggregateBinaryOperator op) {
- return aggregateBinaryOperations(m1, m2, null, op);
- }
-
- @Override
public MatrixBlock aggregateBinaryOperations(MatrixBlock m1,
MatrixBlock m2, MatrixBlock ret,
AggregateBinaryOperator op) {
+ // create output matrix block
return aggregateBinaryOperations(m1, m2, ret, op, false, false);
}
@@ -613,10 +623,10 @@ public class CompressedMatrixBlock extends MatrixBlock {
ret = CLALibLeftMultBy.leftMultByMatrix(this, that,
ret, op.getNumThreads());
}
- if(LOG.isDebugEnabled()){
+ if(LOG.isDebugEnabled()) {
double t = time.stop();
- LOG.debug("MM: Time block w/ sharedDim: " +
m1.getNumColumns() + " rowLeft: " + m1.getNumRows() + " colRight:"
- + m2.getNumColumns() + " in " + t + "ms.");
+ LOG.debug("MM: Time block w/ sharedDim: " +
m1.getNumColumns() + " rowLeft: " + m1.getNumRows()
+ + " colRight:" + m2.getNumColumns() + " in " +
t + "ms.");
}
if(transposeOutput) {
@@ -669,18 +679,15 @@ public class CompressedMatrixBlock extends MatrixBlock {
(op.aggOp.increOp.fn instanceof Builtin &&
(((Builtin)
op.aggOp.increOp.fn).getBuiltinCode() == BuiltinCode.MIN ||
((Builtin)
op.aggOp.increOp.fn).getBuiltinCode() == BuiltinCode.MAX)))) {
- throw new NotImplementedException("Unary aggregate " +
op.aggOp.increOp.fn + " not supported yet.");
+ return getUncompressed("Unary aggregate " +
op.aggOp.increOp.fn + " not supported yet.")
+ .aggregateUnaryOperations(op, result, blen,
indexesIn, inCP);
+
}
return CLALibCompAgg.aggregateUnary(this, result, op, blen,
indexesIn, inCP);
}
@Override
- public MatrixBlock transposeSelfMatrixMultOperations(MatrixBlock out,
MMTSJType tstype) {
- return transposeSelfMatrixMultOperations(out, tstype, 1);
- }
-
- @Override
public MatrixBlock transposeSelfMatrixMultOperations(MatrixBlock out,
MMTSJType tstype, int k) {
// check for transpose type
if(tstype == MMTSJType.LEFT) {
@@ -693,8 +700,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
else
out.reset(clen, clen, false);
out.allocateDenseBlock();
- CLALibLeftMultBy.leftMultByTransposeSelf(_colGroups,
out, k, getNumColumns(), getMaxNumValues(),
- isOverlapping());
+ CLALibLeftMultBy.leftMultByTransposeSelf(this, out, k);
return out;
}
else {
@@ -741,29 +747,6 @@ public class CompressedMatrixBlock extends MatrixBlock {
return null;
}
- public Pair<Integer, int[]> getMaxNumValues() {
- if(v == null) {
-
- int numVals = 1;
- int[] numValues = new int[_colGroups.size()];
- int nr;
- for(int i = 0; i < _colGroups.size(); i++)
- if(_colGroups.get(i) instanceof ColGroupValue) {
- nr = ((ColGroupValue)
_colGroups.get(i)).getNumValues();
- numValues[i] = nr;
- numVals = Math.max(numVals, nr);
- }
- else {
- numValues[i] = -1;
- }
- v = new ImmutablePair<>(numVals, numValues);
- return v;
- }
- else {
- return v;
- }
- }
-
private static class DecompressTask implements Callable<Long> {
private final List<AColGroup> _colGroups;
private final MatrixBlock _ret;
@@ -815,7 +798,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
sb.append("\n" + cg);
}
else
- sb.append("EmptyColGroups");
+ sb.append("\nEmptyColGroups");
return sb.toString();
}
@@ -897,9 +880,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
if(Builtin.isBuiltinCode(op.fn, BuiltinCode.ISNAN,
BuiltinCode.ISNA) && !containsValue(op.getPattern()))
return new MatrixBlock(getNumRows(), getNumColumns(),
0); // avoid unnecessary allocation
- printDecompressWarning("unaryOperations " + op.fn.toString());
- MatrixBlock tmp = getUncompressed();
- return tmp.unaryOperations(op, result);
+ return getUncompressed("unaryOperations " +
op.fn.toString()).unaryOperations(op, result);
}
@Override
@@ -941,6 +922,18 @@ public class CompressedMatrixBlock extends MatrixBlock {
}
@Override
+ public double prod() {
+ AggregateUnaryOperator op =
InstructionUtils.parseBasicAggregateUnaryOperator("ua*", 1);
+ return aggregateUnaryOperations(op, null, 1000,
null).getValue(0, 0);
+ }
+
+ @Override
+ public double mean() {
+ AggregateUnaryOperator op =
InstructionUtils.parseBasicAggregateUnaryOperator("uamean", 1);
+ return aggregateUnaryOperations(op, null, 1000,
null).getValue(0, 0);
+ }
+
+ @Override
public MatrixBlock rexpandOperations(MatrixBlock ret, double max,
boolean rows, boolean cast, boolean ignore,
int k) {
if(rows) {
@@ -960,10 +953,11 @@ public class CompressedMatrixBlock extends MatrixBlock {
@Override
public MatrixBlock binaryOperationsInPlace(BinaryOperator op,
MatrixValue thatValue) {
printDecompressWarning("binaryOperationsInPlace", (MatrixBlock)
thatValue);
- MatrixBlock left = getUncompressed();
+ MatrixBlock left = new MatrixBlock();
+ left.copy(getUncompressed());
MatrixBlock right = getUncompressed(thatValue);
left.binaryOperationsInPlace(op, right);
- return this;
+ return left;
}
@Override
@@ -986,11 +980,6 @@ public class CompressedMatrixBlock extends MatrixBlock {
}
@Override
- public void permutationMatrixMultOperations(MatrixValue m2Val,
MatrixValue out1Val, MatrixValue out2Val) {
- permutationMatrixMultOperations(m2Val, out1Val, out2Val, 1);
- }
-
- @Override
public void permutationMatrixMultOperations(MatrixValue m2Val,
MatrixValue out1Val, MatrixValue out2Val, int k) {
printDecompressWarning("permutationMatrixMultOperations",
(MatrixBlock) m2Val);
MatrixBlock left = getUncompressed();
@@ -1064,60 +1053,23 @@ public class CompressedMatrixBlock extends MatrixBlock {
@Override
public CM_COV_Object covOperations(COVOperator op, MatrixBlock that) {
- printDecompressWarning("covOperations");
- MatrixBlock left = getUncompressed();
MatrixBlock right = getUncompressed(that);
- return left.covOperations(op, right);
+ return getUncompressed("covOperations").covOperations(op,
right);
}
@Override
public CM_COV_Object covOperations(COVOperator op, MatrixBlock that,
MatrixBlock weights) {
- printDecompressWarning("covOperations");
- MatrixBlock left = getUncompressed();
MatrixBlock right1 = getUncompressed(that);
MatrixBlock right2 = getUncompressed(weights);
- return left.covOperations(op, right1, right2);
+ return getUncompressed("covOperations").covOperations(op,
right1, right2);
}
@Override
public MatrixBlock sortOperations(MatrixValue weights, MatrixBlock
result) {
MatrixBlock right = getUncompressed(weights);
- if(_colGroups.size() == 1) {
- AColGroup grp = _colGroups.get(0);
- if(grp instanceof ColGroupEmpty || grp instanceof
ColGroupConst)
- return this;
- printDecompressWarning("sortOperations");
- if(grp instanceof ColGroupUncompressed)
- return
grp.getValuesAsBlock().sortOperations(right, result);
-
- if(right == null && grp instanceof ColGroupCompressed) {
- MatrixBlock vals = grp.getValuesAsBlock();
- int[] counts = ((ColGroupValue)
grp).getCounts();
- double[] data = (vals.getDenseBlock() != null)
? vals.getDenseBlockValues() : null;
- SortUtils.sortByValue(0, vals.getNumRows(),
data, counts);
- MatrixBlock counts2 = getCountsAsBlock(counts);
- if(counts2.isEmpty())
- return vals;
- return vals.sortOperations(counts2, result);
- }
- else
- return getUncompressed().sortOperations(right,
result);
- }
- else {
- printDecompressWarning("sortOperations with multiple
column groups is not supported");
- return getUncompressed().sortOperations(weights,
result);
- }
+ return getUncompressed("sortOperations").sortOperations(right,
result);
}
- // @Override
- // public MatrixBlock aggregateBinaryOperations(MatrixIndexes m1Index,
MatrixBlock m1Value, MatrixIndexes m2Index,
- // MatrixBlock m2Value, MatrixBlock result, AggregateBinaryOperator op)
{
- // if(m2Value == this )
- // return m2Value.aggregateBinaryOperations(m1Value, m2Value, op);
- // else
- // return m1Value.aggregateBinaryOperations(m1Value, m2Value, op);
- // }
-
@Override
public MatrixBlock aggregateTernaryOperations(MatrixBlock m1,
MatrixBlock m2, MatrixBlock m3, MatrixBlock ret,
AggregateTernaryOperator op, boolean inCP) {
@@ -1126,7 +1078,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
boolean m3C = m3 instanceof CompressedMatrixBlock;
printDecompressWarning("aggregateTernaryOperations " +
op.aggOp.getClass().getSimpleName() + " "
+ op.indexFn.getClass().getSimpleName() + " " +
op.aggOp.increOp.fn.getClass().getSimpleName() + " "
- + op.binaryFn.getClass().getSimpleName() + "m1,m2,m3" +
m1C + " " + m2C + " " + m3C);
+ + op.binaryFn.getClass().getSimpleName() + " m1,m2,m3 "
+ m1C + " " + m2C + " " + m3C);
MatrixBlock left = getUncompressed();
MatrixBlock right1 = getUncompressed(m2);
MatrixBlock right2 = getUncompressed(m3);
@@ -1144,12 +1096,6 @@ public class CompressedMatrixBlock extends MatrixBlock {
@Override
public MatrixBlock groupedAggOperations(MatrixValue tgt, MatrixValue
wghts, MatrixValue ret, int ngroups,
- Operator op) {
- return groupedAggOperations(tgt, wghts, ret, ngroups, op, 1);
- }
-
- @Override
- public MatrixBlock groupedAggOperations(MatrixValue tgt, MatrixValue
wghts, MatrixValue ret, int ngroups,
Operator op, int k) {
printDecompressWarning("groupedAggOperations");
MatrixBlock left = getUncompressed();
@@ -1165,13 +1111,6 @@ public class CompressedMatrixBlock extends MatrixBlock {
}
@Override
- public MatrixBlock removeEmptyOperations(MatrixBlock ret, boolean rows,
boolean emptyReturn) {
- printDecompressWarning("removeEmptyOperations");
- MatrixBlock tmp = getUncompressed();
- return tmp.removeEmptyOperations(ret, rows, emptyReturn);
- }
-
- @Override
public void ctableOperations(Operator op, double scalar, MatrixValue
that, CTableMap resultMap,
MatrixBlock resultBlock) {
printDecompressWarning("ctableOperations Var 1");
@@ -1206,27 +1145,18 @@ public class CompressedMatrixBlock extends MatrixBlock {
}
@Override
- public MatrixBlock ctableSeqOperations(MatrixValue that, double scalar,
MatrixBlock resultBlock) {
+ public MatrixBlock ctableSeqOperations(MatrixValue thatMatrix, double
thatScalar, MatrixBlock resultBlock,
+ boolean updateClen) {
printDecompressWarning("ctableOperations Var 5");
MatrixBlock left = getUncompressed();
- MatrixBlock right = getUncompressed(that);
- return left.ctableSeqOperations(right, scalar, resultBlock);
- }
-
- @Override
- public void ctableOperations(Operator op, MatrixValue that, MatrixValue
that2, CTableMap resultMap) {
- printDecompressWarning("ctableOperations Var 6");
- MatrixBlock left = getUncompressed();
- MatrixBlock right1 = getUncompressed(that);
- MatrixBlock right2 = getUncompressed(that2);
- left.ctableOperations(op, right1, right2, resultMap);
+ MatrixBlock right = getUncompressed(thatMatrix);
+ return left.ctableSeqOperations(right, thatScalar, resultBlock,
updateClen);
}
@Override
public void ctableOperations(Operator op, MatrixValue that, MatrixValue
that2, CTableMap resultMap,
MatrixBlock resultBlock) {
- printDecompressWarning("ctableOperations Var 7");
- MatrixBlock left = getUncompressed();
+ MatrixBlock left = getUncompressed("ctableOperations Var 7");
MatrixBlock right1 = getUncompressed(that);
MatrixBlock right2 = getUncompressed(that2);
left.ctableOperations(op, right1, right2, resultMap,
resultBlock);
@@ -1234,8 +1164,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
@Override
public MatrixBlock ternaryOperations(TernaryOperator op, MatrixBlock
m2, MatrixBlock m3, MatrixBlock ret) {
- printDecompressWarning("ternaryOperations " + op.fn);
- MatrixBlock left = getUncompressed();
+ MatrixBlock left = getUncompressed("ternaryOperations " +
op.fn);
MatrixBlock right1 = getUncompressed(m2);
MatrixBlock right2 = getUncompressed(m3);
return left.ternaryOperations(op, right1, right2, ret);
@@ -1243,15 +1172,8 @@ public class CompressedMatrixBlock extends MatrixBlock {
@Override
public MatrixBlock quaternaryOperations(QuaternaryOperator qop,
MatrixBlock um, MatrixBlock vm, MatrixBlock wm,
- MatrixBlock out) {
- return quaternaryOperations(qop, um, vm, wm, out, 1);
- }
-
- @Override
- public MatrixBlock quaternaryOperations(QuaternaryOperator qop,
MatrixBlock um, MatrixBlock vm, MatrixBlock wm,
MatrixBlock out, int k) {
- printDecompressWarning("quaternaryOperations");
- MatrixBlock left = getUncompressed();
+ MatrixBlock left = getUncompressed("quaternaryOperations");
MatrixBlock right1 = getUncompressed(um);
MatrixBlock right2 = getUncompressed(vm);
MatrixBlock right3 = getUncompressed(wm);
@@ -1281,24 +1203,32 @@ public class CompressedMatrixBlock extends MatrixBlock {
}
private static boolean isCompressed(MatrixBlock mb) {
- return(mb instanceof CompressedMatrixBlock);
+ return mb instanceof CompressedMatrixBlock;
}
public static MatrixBlock getUncompressed(MatrixValue mVal) {
+ return isCompressed((MatrixBlock) mVal) ?
((CompressedMatrixBlock) mVal).getUncompressed() : (MatrixBlock) mVal;
+ }
+
+ public static MatrixBlock getUncompressed(MatrixValue mVal, String
message) {
return isCompressed((MatrixBlock) mVal) ?
((CompressedMatrixBlock) mVal)
-
.decompress(OptimizerUtils.getConstrainedNumThreads(-1)) : (MatrixBlock) mVal;
+ .getUncompressed(message) : (MatrixBlock) mVal;
}
public MatrixBlock getUncompressed() {
- return isCompressed((MatrixBlock) this) ?
((CompressedMatrixBlock) this)
-
.decompress(OptimizerUtils.getConstrainedNumThreads(-1)) : (MatrixBlock) this;
+ return
this.decompress(OptimizerUtils.getConstrainedNumThreads(-1));
}
- protected void printDecompressWarning(String operation) {
+ public MatrixBlock getUncompressed(String operation) {
+ printDecompressWarning(operation);
+ return getUncompressed();
+ }
+
+ private void printDecompressWarning(String operation) {
LOG.warn("Operation '" + operation + "' not supported yet -
decompressing for ULA operations.");
}
- protected void printDecompressWarning(String operation, MatrixBlock m2)
{
+ private void printDecompressWarning(String operation, MatrixBlock m2) {
if(isCompressed(m2))
LOG.warn("Operation '" + operation + "' not supported
yet - decompressing for ULA operations.");
else
@@ -1306,11 +1236,6 @@ public class CompressedMatrixBlock extends MatrixBlock {
}
@Override
- public boolean isShallowSerialize() {
- return true;
- }
-
- @Override
public boolean isShallowSerialize(boolean inclConvert) {
return true;
}
@@ -1340,6 +1265,20 @@ public class CompressedMatrixBlock extends MatrixBlock {
copyCompressedMatrix(that);
}
+ @Override
+ public MatrixBlock copyShallow(MatrixBlock that) {
+ if(that instanceof CompressedMatrixBlock)
+ throw new NotImplementedException();
+ else
+ throw new DMLCompressionException(
+ "Invalid copy shallow, since the matrixBlock
given is not of type CompressedMatrixBLock");
+ }
+
+ @Override
+ public void copy(int rl, int ru, int cl, int cu, MatrixBlock src,
boolean awareDestNZ) {
+ throw new DMLCompressionException("Invalid copy into
CompressedMatrixBlock");
+ }
+
private void copyCompressedMatrix(CompressedMatrixBlock that) {
this.rlen = that.rlen;
this.clen = that.clen;
@@ -1354,11 +1293,232 @@ public class CompressedMatrixBlock extends MatrixBlock
{
overlappingColGroups = that.overlappingColGroups;
}
- public SoftReference<MatrixBlock> getSoftReferenceToDecompressed(){
+ public SoftReference<MatrixBlock> getSoftReferenceToDecompressed() {
return decompressedVersion;
}
- public void clearSoftReferenceToDecompressed(){
+ public void clearSoftReferenceToDecompressed() {
decompressedVersion = null;
}
+
+ @Override
+ public DenseBlock getDenseBlock() {
+ throw new DMLCompressionException("Should not get DenseBlock on
a compressed Matrix");
+ }
+
+ @Override
+ public void setDenseBlock(DenseBlock dblock) {
+ throw new DMLCompressionException("Should not set DenseBlock on
a compressed Matrix");
+ }
+
+ @Override
+ public double[] getDenseBlockValues() {
+ throw new DMLCompressionException("Should not get DenseBlock
values on a compressed Matrix");
+ }
+
+ @Override
+ public SparseBlock getSparseBlock() {
+ throw new DMLCompressionException("Should not get SparseBlock
on a compressed Matrix");
+ }
+
+ @Override
+ public void setSparseBlock(SparseBlock sblock) {
+ throw new DMLCompressionException("Should not set SparseBlock
on a compressed Matrix");
+ }
+
+ @Override
+ public Iterator<IJV> getSparseBlockIterator() {
+ throw new DMLCompressionException("Should not get
SparseBlockIterator on a compressed Matrix");
+ }
+
+ @Override
+ public Iterator<IJV> getSparseBlockIterator(int rl, int ru) {
+ throw new DMLCompressionException("Should not get
SparseBlockIterator on a compressed Matrix");
+ }
+
+ @Override
+ public void quickSetValue(int r, int c, double v) {
+ throw new DMLCompressionException("Should not set a value on a
compressed Matrix");
+ }
+
+ @Override
+ public void quickSetValueThreadSafe(int r, int c, double v) {
+ throw new DMLCompressionException("Thread safe execution does
not work on Compressed Matrix");
+ }
+
+ @Override
+ public double quickGetValueThreadSafe(int r, int c) {
+ throw new DMLCompressionException("Thread safe execution does
not work on Compressed Matrix");
+ }
+
+ @Override
+ public double getValueDenseUnsafe(int r, int c) {
+ throw new DMLCompressionException("Compressed Matrix does not
have a dense matrix block");
+ }
+
+ @Override
+ public void appendValue(int r, int c, double v) {
+ throw new DMLCompressionException("Cant append value to
compressed Matrix");
+ }
+
+ @Override
+ public void appendValuePlain(int r, int c, double v) {
+ throw new DMLCompressionException("Can't append value to
compressed Matrix");
+ }
+
+ @Override
+ public void appendRow(int r, SparseRow row, boolean deep) {
+ throw new DMLCompressionException("Can't append row to
compressed Matrix");
+ }
+
+ @Override
+ public void appendToSparse(MatrixBlock that, int rowoffset, int
coloffset, boolean deep) {
+ throw new DMLCompressionException("Can't append to compressed
Matrix");
+ }
+
+ @Override
+ public void appendRowToSparse(SparseBlock dest, MatrixBlock src, int i,
int rowoffset, int coloffset,
+ boolean deep) {
+ throw new DMLCompressionException("Can't append row to
compressed Matrix");
+ }
+
+ @Override
+ public void sortSparseRows() {
+ throw new DMLCompressionException("It does not make sense to
sort the rows in a compressed matrix");
+ }
+
+ @Override
+ public void sortSparseRows(int rl, int ru) {
+ throw new DMLCompressionException("It does not make sense to
sort the rows in a compressed matrix");
+ }
+
+ @Override
+ public double minNonZero() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean isInSparseFormat() {
+ return false;
+ }
+
+ @Override
+ public boolean isUltraSparse() {
+ return false;
+ }
+
+ @Override
+ public boolean isUltraSparse(boolean checkNnz) {
+ return false;
+ }
+
+ @Override
+ public boolean isSparsePermutationMatrix() {
+ return false;
+ }
+
+ @Override
+ public boolean evalSparseFormatInMemory() {
+ return false;
+ }
+
+ @Override
+ public boolean evalSparseFormatOnDisk() {
+ return false;
+ }
+
+ @Override
+ public void examSparsity(boolean allowCSR) {
+ // do nothing
+ }
+
+ @Override
+ public void sparseToDense() {
+ // do nothing
+ }
+
+ @Override
+ public void merge(MatrixBlock that, boolean appendOnly, boolean par,
boolean deep) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void compactEmptyBlock() {
+ // do nothing
+ }
+
+ @Override
+ public void dropLastRowsOrColumns(CorrectionLocationType
correctionLocation) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public double interQuartileMean() {
+ return getUncompressed("interQuartileMean").interQuartileMean();
+ }
+
+ @Override
+ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret) {
+ return getUncompressed("pickValues").pickValues(quantiles, ret);
+ }
+
+ @Override
+ public double pickValue(double quantile, boolean average) {
+ return getUncompressed("pickValue").pickValue(quantile,
average);
+ }
+
+ @Override
+ public double sumWeightForQuantile() {
+ return
getUncompressed("sumWeightForQuantile").sumWeightForQuantile();
+ }
+
+ @Override
+ public MatrixBlock extractTriangular(MatrixBlock ret, boolean lower,
boolean diag, boolean values) {
+ return
getUncompressed("extractTriangular").extractTriangular(ret, lower, diag,
values);
+ }
+
+ @Override
+ public boolean isThreadSafe() {
+ return false;
+ }
+
+ @Override
+ public void checkNaN() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void init(double[][] arr, int r, int c) {
+ throw new DMLCompressionException("Invalid to init on a
compressed MatrixBlock");
+ }
+
+ @Override
+ public void init(double[] arr, int r, int c) {
+ throw new DMLCompressionException("Invalid to init on a
compressed MatrixBlock");
+ }
+
+ @Override
+ public boolean isAllocated() {
+ return true;
+ }
+
+ @Override
+ public Future<MatrixBlock> allocateBlockAsync() {
+ throw new DMLCompressionException("Invalid to allocate dense
block on a compressed MatrixBlock");
+ }
+
+ @Override
+ public boolean allocateDenseBlock(boolean clearNNZ) {
+ throw new DMLCompressionException("Invalid to allocate dense
block on a compressed MatrixBlock");
+ }
+
+ @Override
+ public boolean allocateSparseRowsBlock(boolean clearNNZ) {
+ throw new DMLCompressionException("Invalid to allocate sparse
block on a compressed MatrixBlock");
+ }
+
+ @Override
+ public void allocateAndResetSparseBlock(boolean clearNNZ,
SparseBlock.Type stype) {
+ throw new DMLCompressionException("Invalid to allocate block on
a compressed MatrixBlock");
+ }
}
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java
b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java
index 7a8b827..04ccdc2 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java
@@ -28,7 +28,6 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.DMLRuntimeException;
@@ -50,7 +49,7 @@ public class CLALibLeftMultBy {
if(m2.isEmpty())
return ret;
MatrixBlock transposed = new MatrixBlock(m2.getNumColumns(),
m2.getNumRows(), false);
- LibMatrixReorg.transpose(m2, transposed);
+ LibMatrixReorg.transpose(m2, transposed, k);
ret = leftMultByMatrix(m1, transposed, ret, k);
ret.recomputeNonZeros();
return ret;
@@ -59,28 +58,28 @@ public class CLALibLeftMultBy {
public static MatrixBlock
leftMultByMatrixTransposed(CompressedMatrixBlock m1, CompressedMatrixBlock m2,
MatrixBlock ret, int k) {
- prepareReturnMatrix(m1, m2, ret, true);
- leftMultByCompressedTransposedMatrix(m1.getColGroups(), m2,
ret, k, m1.getNumColumns(), m1.getMaxNumValues(),
- m1.isOverlapping());
+ ret = prepareReturnMatrix(m1, m2, ret, true);
+ leftMultByCompressedTransposedMatrix(m1.getColGroups(), m2,
ret, k, m1.getNumColumns(), m1.isOverlapping());
ret.recomputeNonZeros();
return ret;
}
public static MatrixBlock leftMultByMatrix(CompressedMatrixBlock m1,
MatrixBlock m2, MatrixBlock ret, int k) {
- prepareReturnMatrix(m1, m2, ret, false);
+ ret = prepareReturnMatrix(m1, m2, ret, false);
+
if(m2.isEmpty())
return ret;
- ret = leftMultByMatrix(m1.getColGroups(), m2, ret, k,
m1.getNumColumns(), m1.getMaxNumValues().getLeft(),
- m1.isOverlapping());
+
+ ret = leftMultByMatrix(m1.getColGroups(), m2, ret, k,
m1.getNumColumns(), m1.isOverlapping());
ret.recomputeNonZeros();
return ret;
}
private static MatrixBlock prepareReturnMatrix(MatrixBlock m1,
MatrixBlock m2, MatrixBlock ret,
boolean doTranspose) {
- int numRowsOutput = doTranspose ? m2.getNumColumns() :
m2.getNumRows();
- int numColumnsOutput = m1.getNumColumns();
+ final int numRowsOutput = doTranspose ? m2.getNumColumns() :
m2.getNumRows();
+ final int numColumnsOutput = m1.getNumColumns();
if(ret == null)
ret = new MatrixBlock(numRowsOutput, numColumnsOutput,
false, numRowsOutput * numColumnsOutput);
else if(!(ret.getNumColumns() == numColumnsOutput &&
ret.getNumRows() == numRowsOutput && ret.isAllocated()))
@@ -88,11 +87,12 @@ public class CLALibLeftMultBy {
return ret;
}
- public static void leftMultByTransposeSelf(List<AColGroup> groups,
MatrixBlock result, int k, int numColumns,
- Pair<Integer, int[]> v, boolean overlapping) {
-
+ public static void leftMultByTransposeSelf(CompressedMatrixBlock cmb,
MatrixBlock result, int k) {
+ final int numColumns = cmb.getNumColumns();
+ final boolean overlapping = cmb.isOverlapping();
+ List<AColGroup> groups = cmb.getColGroups();
result.allocateDenseBlock();
-
+
if(overlapping) {
LOG.warn("Inefficient TSMM with overlapping matrix
could be implemented multi-threaded but is not yet.");
leftMultByCompressedTransposedMatrix(groups, groups,
result);
@@ -105,10 +105,12 @@ public class CLALibLeftMultBy {
try {
ExecutorService pool = CommonThreadPool.get(k);
ArrayList<Callable<Object>> tasks = new
ArrayList<>();
+
for(int i = 0; i < groups.size(); i++) {
final AColGroup g = groups.get(i);
tasks.add(new
LeftMultByCompressedTransposedMatrixTask(groups, g, result, i, groups.size()));
}
+
for(Future<Object> tret : pool.invokeAll(tasks))
tret.get();
pool.shutdown();
@@ -137,8 +139,7 @@ public class CLALibLeftMultBy {
}
private static MatrixBlock
leftMultByCompressedTransposedMatrix(List<AColGroup> colGroups,
- CompressedMatrixBlock that, MatrixBlock ret, int k, int
numColumns, Pair<Integer, int[]> v,
- boolean overlapping) {
+ CompressedMatrixBlock that, MatrixBlock ret, int k, int
numColumns, boolean overlapping) {
ret.allocateDenseBlock();
List<AColGroup> thatCGs = that.getColGroups();
@@ -227,7 +228,7 @@ public class CLALibLeftMultBy {
}
private static MatrixBlock leftMultByMatrix(List<AColGroup> colGroups,
MatrixBlock that, MatrixBlock ret, int k,
- int numColumns, int maxNumValues, boolean overlapping) {
+ int numColumns, boolean overlapping) {
if(that.isEmpty()) {
ret.setNonZeros(0);
@@ -251,7 +252,7 @@ public class CLALibLeftMultBy {
tmpRet.allocateDenseBlock();
for(int blo = 0; blo <
that.getNumRows(); blo += rowBlockSize)
tasks.add(new
LeftMatrixColGroupMultTaskOld(g, that, tmpRet, blo,
- Math.min(blo +
rowBlockSize, that.getNumRows()), maxNumValues));
+ Math.min(blo +
rowBlockSize, that.getNumRows())));
}
List<Future<MatrixBlock>> futures =
pool.invokeAll(tasks);
@@ -269,7 +270,7 @@ public class CLALibLeftMultBy {
}
else {
- List<List<AColGroup>> split =
split(colGroups, Math.max(k / 2 / that.getNumRows(), 1));
+ List<List<AColGroup>> split =
split(colGroups, Math.max(k / that.getNumRows(), 1));
for(int blo = 0; blo <
that.getNumRows(); blo += rowBlockSize) {
for(List<AColGroup> gr
: split)
tasks.add(new
LeftMatrixColGroupMultTaskNew(gr, that, ret, numColumns, blo,
@@ -316,23 +317,18 @@ public class CLALibLeftMultBy {
private final MatrixBlock _ret;
private final int _rl;
private final int _ru;
- private final int _maxNrValues;
- protected LeftMatrixColGroupMultTaskOld(AColGroup group,
MatrixBlock that, MatrixBlock ret, int rl, int ru,
- int maxNrValues) {
+ protected LeftMatrixColGroupMultTaskOld(AColGroup group,
MatrixBlock that, MatrixBlock ret, int rl, int ru) {
_group = group;
_that = that;
_ret = ret;
_rl = rl;
_ru = ru;
- _maxNrValues = maxNrValues;
}
@Override
public MatrixBlock call() {
try {
-
ColGroupValue.setupThreadLocalMemory(_maxNrValues * (_ru - _rl));
-
ColGroupValue.setupLeftMultThreadLocalMemory(_group.getNumCols() * (_ru - _rl));
_group.leftMultByMatrix(_that, _ret, _rl, _ru);
}
catch(Exception e) {
@@ -383,18 +379,16 @@ public class CLALibLeftMultBy {
}
}
else {
-
List<ColGroupValue> v = new ArrayList<>();
int rowBlockSize = 1;
- int colBlockSize = 4000;
- int colGroupBlocking = 4;
List<MatrixBlock> preAgg = new ArrayList<>();
+ int colGroupBlocking = 16;
for(int j = 0; j < colGroupBlocking; j++) {
MatrixBlock m = new MatrixBlock(1, 1, false);
m.allocateDenseBlock();
preAgg.add(m);
-
}
+
MatrixBlock tmpRes = new MatrixBlock(rowBlockSize,
numColumns, false);
for(int j = 0; j < colGroups.size(); j++) {
@@ -411,8 +405,11 @@ public class CLALibLeftMultBy {
for(int g = 0; g < v.size(); g += colGroupBlocking) {
final int gEnd = Math.min(g + colGroupBlocking,
colGroups.size());
for(int j = g; j < gEnd && j < v.size(); j++) {
- preAgg.get(j %
colGroupBlocking).reset(rowBlockSize, v.get(j).getNumValues(), false);
+ ColGroupValue cg = v.get(j);
+ preAgg.get(j %
colGroupBlocking).reset(rowBlockSize, cg.getNumValues(), false);
}
+ // int colBlockSize = 16000;
+ int colBlockSize = 64000;
for(int h = rl; h < ru; h += rowBlockSize) {
for(int i = 0; i <
that.getNumColumns(); i += colBlockSize) {
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java
b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java
index d486fa8..edc68b5 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java
@@ -28,7 +28,6 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.DMLRuntimeException;
@@ -43,20 +42,20 @@ public class CLALibRightMultBy {
public static MatrixBlock rightMultByMatrix(CompressedMatrixBlock m1,
MatrixBlock m2, MatrixBlock ret, int k,
boolean allowOverlap) {
- ret = rightMultByMatrix(m1.getColGroups(), m2, ret, k,
m1.getMaxNumValues(), allowOverlap);
+ ret = rightMultByMatrix(m1.getColGroups(), m2, ret, k,
allowOverlap);
ret.recomputeNonZeros();
return ret;
}
private static MatrixBlock rightMultByMatrix(List<AColGroup> colGroups,
MatrixBlock that, MatrixBlock ret, int k,
- Pair<Integer, int[]> v, boolean allowOverlap) {
+ boolean allowOverlap) {
if(that instanceof CompressedMatrixBlock)
LOG.warn("Decompression Right matrix");
that = that instanceof CompressedMatrixBlock ?
((CompressedMatrixBlock) that).decompress(k) : that;
- MatrixBlock m = rightMultByMatrixOverlapping(colGroups, that,
ret, k, v);
+ MatrixBlock m = rightMultByMatrixOverlapping(colGroups, that,
ret, k);
if(m instanceof CompressedMatrixBlock)
if(allowOverlappingOutput(colGroups, allowOverlap))
@@ -94,18 +93,18 @@ public class CLALibRightMultBy {
}
private static MatrixBlock rightMultByMatrixOverlapping(List<AColGroup>
colGroups, MatrixBlock that,
- MatrixBlock ret, int k, Pair<Integer, int[]> v) {
+ MatrixBlock ret, int k) {
int rl = colGroups.get(0).getNumRows();
int cl = that.getNumColumns();
// Create an overlapping compressed Matrix Block.
ret = new CompressedMatrixBlock(rl, cl);
CompressedMatrixBlock retC = (CompressedMatrixBlock) ret;
- ret = rightMultByMatrixCompressed(colGroups, that, retC, k, v);
+ ret = rightMultByMatrixCompressed(colGroups, that, retC, k);
return ret;
}
private static MatrixBlock rightMultByMatrixCompressed(List<AColGroup>
colGroups, MatrixBlock that,
- CompressedMatrixBlock ret, int k, Pair<Integer, int[]> v) {
+ CompressedMatrixBlock ret, int k) {
List<AColGroup> retCg = new ArrayList<>();
boolean containsNull = false;
diff --git
a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
index 77e949e..8d2d672 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
@@ -47,6 +47,7 @@ import org.apache.sysds.lops.MMTSJ.MMTSJType;
import org.apache.sysds.lops.MapMultChain.ChainType;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
+import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.lib.CLALibBinaryCellOp;
import org.apache.sysds.runtime.controlprogram.caching.CacheBlock;
import org.apache.sysds.runtime.controlprogram.caching.LazyWriteBuffer;
@@ -115,6 +116,7 @@ import org.apache.sysds.utils.NativeHelper;
public class MatrixBlock extends MatrixValue implements CacheBlock,
Externalizable {
+ // private static final Log LOG =
LogFactory.getLog(MatrixBlock.class.getName());
private static final long serialVersionUID = 7319972089143154056L;
@@ -151,7 +153,6 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
////////
// Matrix Constructors
//
-
public MatrixBlock() {
this(0, 0, true, -1);
}
@@ -220,36 +221,41 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
denseBlock = dBlock;
}
+
+ protected MatrixBlock(boolean empty){
+ // do nothing
+ }
+
////////
// Initialization methods
// (reset, init, allocate, etc)
@Override
- public void reset() {
+ public final void reset() {
reset(rlen, clen, sparse, -1, 0);
}
@Override
- public void reset(int rl, int cl) {
+ public final void reset(int rl, int cl) {
reset(rl, cl, sparse, -1, 0);
}
- public void reset(int rl, int cl, long estnnz) {
+ public final void reset(int rl, int cl, long estnnz) {
reset(rl, cl, evalSparseFormatInMemory(rl, cl, estnnz), estnnz,
0);
}
@Override
- public void reset(int rl, int cl, boolean sp) {
+ public final void reset(int rl, int cl, boolean sp) {
reset(rl, cl, sp, -1, 0);
}
@Override
- public void reset(int rl, int cl, boolean sp, long estnnz) {
+ public final void reset(int rl, int cl, boolean sp, long estnnz) {
reset(rl, cl, sp, estnnz, 0);
}
@Override
- public void reset(int rl, int cl, double val) {
+ public final void reset(int rl, int cl, double val) {
reset(rl, cl, false, -1, val);
}
@@ -262,7 +268,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
* @param estnnz estimated number of non-zeros
* @param val initialization value
*/
- private void reset(int rl, int cl, boolean sp, long estnnz, double val)
{
+ public void reset(int rl, int cl, boolean sp, long estnnz, double val) {
//check for valid dimensions
if( rl < 0 || cl < 0 )
throw new RuntimeException("Invalid block dimensions:
"+rl+" "+cl);
@@ -349,7 +355,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
return sparse ? (sparseBlock!=null) : (denseBlock!=null);
}
- public MatrixBlock allocateDenseBlock() {
+ public final MatrixBlock allocateDenseBlock() {
allocateDenseBlock( true );
return this;
}
@@ -360,7 +366,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
ConcurrentUtils.constantFuture(allocateBlock());
//fallback sync
}
- public MatrixBlock allocateBlock() {
+ public final MatrixBlock allocateBlock() {
if( sparse )
allocateSparseRowsBlock();
else
@@ -385,7 +391,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
return reset;
}
- public boolean allocateSparseRowsBlock() {
+ public final boolean allocateSparseRowsBlock() {
return allocateSparseRowsBlock(true);
}
@@ -429,7 +435,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
* @param rl number of rows
* @param cl number of columns
*/
- public void allocateDenseBlockUnsafe(int rl, int cl) {
+ public final void allocateDenseBlockUnsafe(int rl, int cl) {
sparse=false;
rlen=rl;
clen=cl;
@@ -447,7 +453,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
* @param dense if true, set dense block to null
* @param sparse if true, set sparse block to null
*/
- public void cleanupBlock( boolean dense, boolean sparse ) {
+ public final void cleanupBlock( boolean dense, boolean sparse ) {
if(dense)
denseBlock = null;
if(sparse)
@@ -458,7 +464,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
// Metadata information
@Override
- public int getNumRows() {
+ public final int getNumRows() {
return rlen;
}
@@ -468,51 +474,51 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
*
* @param r number of rows
*/
- public void setNumRows(int r) {
+ public final void setNumRows(int r) {
rlen = r;
}
@Override
- public int getNumColumns() {
+ public final int getNumColumns() {
return clen;
}
- public void setNumColumns(int c) {
+ public final void setNumColumns(int c) {
clen = c;
}
@Override
- public long getNonZeros() {
+ public final long getNonZeros() {
return nonZeros;
}
- public long setNonZeros(long nnz) {
+ public final long setNonZeros(long nnz) {
return (nonZeros = nnz);
}
- public double getSparsity() {
+ public final double getSparsity() {
return OptimizerUtils.getSparsity(rlen, clen, nonZeros);
}
@Override
- public DataCharacteristics getDataCharacteristics() {
+ public final DataCharacteristics getDataCharacteristics() {
return new MatrixCharacteristics(rlen, clen, -1, nonZeros);
}
- public boolean isVector() {
+ public final boolean isVector() {
return (rlen == 1 || clen == 1);
}
- public long getLength() {
+ public final long getLength() {
return (long)rlen * clen;
}
@Override
- public boolean isEmpty() {
+ public final boolean isEmpty() {
return isEmptyBlock(false);
}
- public boolean isEmptyBlock() {
+ public final boolean isEmptyBlock() {
return isEmptyBlock(true);
}
@@ -648,10 +654,14 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
}
}
- /*
- Thread save set.
- Blocks need to be allocated, and in case of MCSR sparse, all
rows
- that are going to be accessed need to be allocated as well.
+ /**
+ * Thread save set.
+ * Blocks need to be allocated, and in case of MCSR sparse, all rows
+ * that are going to be accessed need to be allocated as well.
+ *
+ * @param r row
+ * @param c column
+ * @param v value
*/
public void quickSetValueThreadSafe(int r, int c, double v) {
if(sparse) {
@@ -965,6 +975,16 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
}
/**
+ * Wrapper method for reduceall-colSum of a matrix.
+ *
+ * @return A new MatrixBlock containing the column sums of this matrix.
+ */
+ public MatrixBlock colSum() {
+ AggregateUnaryOperator op =
InstructionUtils.parseBasicAggregateUnaryOperator("uack+", 1);
+ return aggregateUnaryOperations(op, null, 1000, null);
+ }
+
+ /**
* Wrapper method for reduceall-sumSq of a matrix.
*
* @return Sum of the squared values in the matrix.
@@ -1332,50 +1352,48 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
return 0; //empty block
}
-
+
/**
- * Basic debugging primitive to check correctness of nnz.
- * This method is not intended for production use.
+ * Basic debugging primitive to check correctness of nnz. This method
is not intended for production use.
*/
public void checkNonZeros() {
- //take non-zeros before and after recompute nnz
+ // take non-zeros before and after recompute nnz
long nnzBefore = getNonZeros();
recomputeNonZeros();
long nnzAfter = getNonZeros();
-
- //raise exception if non-zeros don't match up
- if( nnzBefore != nnzAfter )
- throw new RuntimeException("Number of non zeros
incorrect: "+nnzBefore+" vs "+nnzAfter);
+
+ // raise exception if non-zeros don't match up
+ if(nnzBefore != nnzAfter)
+ throw new RuntimeException("Number of non zeros
incorrect: " + nnzBefore + " vs " + nnzAfter);
}
-
+
public void checkSparseRows() {
checkSparseRows(0, rlen);
}
-
+
/**
- * Basic debugging primitive to check sparse block column ordering.
- * This method is not intended for production use.
+ * Basic debugging primitive to check sparse block column ordering.
This method is not intended for production use.
*
* @param rl row lower bound (inclusive)
* @param ru row upper bound (exclusive)
*/
public void checkSparseRows(int rl, int ru) {
- if( !sparse || sparseBlock == null )
+ if(!sparse || sparseBlock == null)
return;
-
- //check ordering of column indexes per sparse row
- for( int i=rl; i<ru; i++ )
- if( !sparseBlock.isEmpty(i) ) {
+
+ // check ordering of column indexes per sparse row
+ for(int i = rl; i < ru; i++)
+ if(!sparseBlock.isEmpty(i)) {
int apos = sparseBlock.pos(i);
int alen = sparseBlock.size(i);
int[] aix = sparseBlock.indexes(i);
double[] avals = sparseBlock.values(i);
- for( int k=apos+1; k<apos+alen; k++ )
- if( aix[k-1] >= aix[k] )
- throw new
RuntimeException("Wrong sparse row ordering: "+k+" "+aix[k-1]+" "+aix[k]);
- for( int k=apos; k<apos+alen; k++ )
- if( avals[k] == 0 )
- throw new
RuntimeException("Wrong sparse row: zero at "+k);
+ for(int k = apos + 1; k < apos + alen; k++)
+ if(aix[k - 1] >= aix[k])
+ throw new
RuntimeException("Wrong sparse row ordering: " + k + " " + aix[k - 1] + " " +
aix[k]);
+ for(int k = apos; k < apos + alen; k++)
+ if(avals[k] == 0)
+ throw new
RuntimeException("Wrong sparse row: zero at " + k);
}
}
@@ -1390,9 +1408,10 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
public void copy(MatrixValue thatValue, boolean sp)
{
MatrixBlock that = checkType(thatValue);
-
if( this == that ) //prevent data loss (e.g., on sparse-dense
conversion)
throw new RuntimeException( "Copy must not overwrite
itself!" );
+ if(that instanceof CompressedMatrixBlock)
+ that = CompressedMatrixBlock.getUncompressed(that,
"Copy not effecient into a MatrixBlock");
rlen=that.rlen;
clen=that.clen;
@@ -1409,6 +1428,8 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
}
public MatrixBlock copyShallow(MatrixBlock that) {
+ if(that instanceof CompressedMatrixBlock)
+ throw new DMLCompressionException("Invalid copy from
CompressedMatrixBlock");
rlen = that.rlen;
clen = that.clen;
nonZeros = that.nonZeros;
@@ -2893,11 +2914,17 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
sparseUnaryOperations(op, ret);
}
+ public final MatrixBlock binaryOperations(BinaryOperator op,
MatrixValue thatValue){
+ return binaryOperations(op, thatValue, null);
+ }
+
@Override
public MatrixBlock binaryOperations(BinaryOperator op, MatrixValue
thatValue, MatrixValue result) {
MatrixBlock that = checkType(thatValue);
MatrixBlock ret = checkType(result);
LibMatrixBincell.isValidDimensionsBinary(this, that);
+ if(thatValue instanceof CompressedMatrixBlock)
+ return ((CompressedMatrixBlock)
thatValue).binaryOperationsLeft(op, this, result);
//compute output dimensions
boolean outer = (LibMatrixBincell.getBinaryAccessType(this,
that)
@@ -2939,6 +2966,11 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
}
public MatrixBlock ternaryOperations(TernaryOperator op, MatrixBlock
m2, MatrixBlock m3, MatrixBlock ret) {
+ if(m2 instanceof CompressedMatrixBlock)
+ m2 = ((CompressedMatrixBlock)
m2).getUncompressed("Ternay Operator arg2 " + op.fn.getClass().getSimpleName());
+ if(m3 instanceof CompressedMatrixBlock)
+ m3 = ((CompressedMatrixBlock)
m3).getUncompressed("Ternay Operator arg3 " + op.fn.getClass().getSimpleName());
+
//prepare inputs
final boolean s1 = (rlen==1 && clen==1);
final boolean s2 = (m2.rlen==1 && m2.clen==1);
@@ -3516,19 +3548,20 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
return result;
}
- public MatrixBlock append( MatrixBlock that, MatrixBlock ret ) {
+ public final MatrixBlock append( MatrixBlock that, MatrixBlock ret ) {
return append(that, ret, true); //default cbind
}
- public MatrixBlock append( MatrixBlock that, MatrixBlock ret, boolean
cbind ) {
+ public final MatrixBlock append( MatrixBlock that, MatrixBlock ret,
boolean cbind ) {
return append(new MatrixBlock[]{that}, ret, cbind);
}
- public MatrixBlock append( MatrixBlock[] that, MatrixBlock ret, boolean
cbind ) {
- MatrixBlock result = checkType( ret );
- final int m = cbind ? rlen :
rlen+Arrays.stream(that).mapToInt(mb -> mb.rlen).sum();
- final int n = cbind ? clen+Arrays.stream(that).mapToInt(mb ->
mb.clen).sum() : clen;
- final long nnz = nonZeros+Arrays.stream(that).mapToLong(mb ->
mb.nonZeros).sum();
+ public MatrixBlock append( MatrixBlock[] that, MatrixBlock result,
boolean cbind) {
+ checkDimensionsForAppend(that, cbind);
+
+ final int m = cbind ? rlen : rlen +
Arrays.stream(that).mapToInt(mb -> mb.rlen).sum();
+ final int n = cbind ? clen + Arrays.stream(that).mapToInt(mb ->
mb.clen).sum() : clen;
+ final long nnz = nonZeros + Arrays.stream(that).mapToLong(mb ->
mb.nonZeros).sum();
boolean shallowCopy = (nonZeros == nnz);
boolean sp = evalSparseFormatInMemory(m, n, nnz);
@@ -3616,6 +3649,21 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
result.nonZeros = nnz;
return result;
}
+
+ public void checkDimensionsForAppend(MatrixBlock[] in, boolean cbind) {
+ if(cbind) {
+ for(int i = 0; i < in.length; i++)
+ if(in[i].rlen != rlen)
+ throw new DMLRuntimeException(
+ "Invalid nRow dimension for
append cbind: was " + in[i].rlen + " should be: " + rlen);
+ }
+ else {
+ for(int i = 0; i < in.length; i++)
+ if(in[i].clen != clen)
+ throw new DMLRuntimeException(
+ "Invalid nCol dimension for
append rbind: was " + in[i].clen + " should be: " + clen);
+ }
+ }
public static MatrixBlock naryOperations(Operator op, MatrixBlock[]
matrices, ScalarObject[] scalars, MatrixBlock ret) {
//note: currently only min max, plus supported and hence
specialized implementation
@@ -3976,24 +4024,24 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
* @param ru The row lower to end at
* @return The sliced out matrix block.
*/
- public MatrixBlock slice(int rl, int ru) {
+ public final MatrixBlock slice(int rl, int ru) {
return slice(rl, ru, 0, clen-1, true, null);
}
- public MatrixBlock slice(int rl, int ru, boolean deep){
+ public final MatrixBlock slice(int rl, int ru, boolean deep){
return slice(rl,ru, 0, clen-1, deep, null);
}
- public MatrixBlock slice(int rl, int ru, int cl, int cu){
+ public final MatrixBlock slice(int rl, int ru, int cl, int cu){
return slice(rl, ru, cl, cu, true, null);
}
@Override
- public MatrixBlock slice(int rl, int ru, int cl, int cu, CacheBlock
ret) {
+ public final MatrixBlock slice(int rl, int ru, int cl, int cu,
CacheBlock ret) {
return slice(rl, ru, cl, cu, true, ret);
}
- public MatrixBlock slice(int rl, int ru, int cl, int cu, boolean deep){
+ public final MatrixBlock slice(int rl, int ru, int cl, int cu, boolean
deep){
return slice(rl, ru, cl, cu, deep, null);
}
@@ -4381,7 +4429,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
}
@Override
- public MatrixBlock aggregateUnaryOperations(AggregateUnaryOperator op,
+ public final MatrixBlock
aggregateUnaryOperations(AggregateUnaryOperator op,
MatrixValue result, int blen, MatrixIndexes indexesIn) {
return aggregateUnaryOperations(op, result, blen, indexesIn,
false);
}
@@ -4390,7 +4438,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
public MatrixBlock aggregateUnaryOperations(AggregateUnaryOperator op,
MatrixValue result,
int blen, MatrixIndexes indexesIn, boolean inCP) {
- MatrixBlock ret = prepareAggregateUnaryOutput(op,result,blen);
+ MatrixBlock ret = prepareAggregateUnaryOutput(op, result, blen);
if( LibMatrixAgg.isSupportedUnaryAggregateOperator(op) ) {
if( op.getNumThreads() > 1 )
@@ -4405,7 +4453,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
denseAggregateUnaryHelp(op, ret, blen, indexesIn);
if(op.aggOp.existsCorrection() && inCP)
-
((MatrixBlock)result).dropLastRowsOrColumns(op.aggOp.correction);
+ ret.dropLastRowsOrColumns(op.aggOp.correction);
return ret;
}
@@ -4777,6 +4825,15 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
return covobj;
}
+
+ public final MatrixBlock sortOperations(){
+ return sortOperations(null, null);
+ }
+
+ public final MatrixBlock sortOperations(MatrixValue weights){
+ return sortOperations(weights, null);
+ }
+
public MatrixBlock sortOperations(MatrixValue weights, MatrixBlock
result) {
boolean wtflag = (weights!=null);
@@ -4904,7 +4961,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
return pickValue(0.5, sum_wt%2==0);
}
- public double pickValue(double quantile){
+ public final double pickValue(double quantile){
return pickValue(quantile, false);
}
@@ -4971,7 +5028,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
return sum_wt;
}
- public MatrixBlock aggregateBinaryOperations(MatrixBlock m1,
MatrixBlock m2, AggregateBinaryOperator op){
+ public final MatrixBlock aggregateBinaryOperations(MatrixBlock m1,
MatrixBlock m2, AggregateBinaryOperator op){
return aggregateBinaryOperations(m1, m2, null, op);
}
@@ -5008,6 +5065,13 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
public MatrixBlock aggregateTernaryOperations(MatrixBlock m1,
MatrixBlock m2, MatrixBlock m3, MatrixBlock ret,
AggregateTernaryOperator op, boolean inCP) {
+ if(m1 instanceof CompressedMatrixBlock)
+ m1 = ((CompressedMatrixBlock)
m1).getUncompressed("Aggregate Ternay Operator arg1 " +
op.getClass().getSimpleName());
+ if(m2 instanceof CompressedMatrixBlock)
+ m2 = ((CompressedMatrixBlock)
m2).getUncompressed("Aggregate Ternay Operator arg2 " +
op.getClass().getSimpleName());
+ if(m3 instanceof CompressedMatrixBlock)
+ m3 = ((CompressedMatrixBlock)
m3).getUncompressed("Aggregate Ternay Operator arg3 " +
op.getClass().getSimpleName());
+
//create output matrix block w/ corrections
int rl = (op.indexFn instanceof ReduceRow) ? 2 : 1;
int cl = (op.indexFn instanceof ReduceRow) ? m1.clen : 2;
@@ -5073,7 +5137,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
* @param op operator
* @return matrix block
*/
- public MatrixBlock groupedAggOperations(MatrixValue tgt, MatrixValue
wghts, MatrixValue ret, int ngroups, Operator op) {
+ public final MatrixBlock groupedAggOperations(MatrixValue tgt,
MatrixValue wghts, MatrixValue ret, int ngroups, Operator op) {
//single-threaded grouped aggregate
return groupedAggOperations(tgt, wghts, ret, ngroups, op, 1);
}
@@ -5127,7 +5191,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
return LibMatrixReorg.rmempty(this, ret, rows, emptyReturn,
select);
}
- public MatrixBlock removeEmptyOperations( MatrixBlock ret, boolean
rows, boolean emptyReturn) {
+ public final MatrixBlock removeEmptyOperations( MatrixBlock ret,
boolean rows, boolean emptyReturn) {
return removeEmptyOperations(ret, rows, emptyReturn, null);
}
@@ -5378,6 +5442,9 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
{
//setup ctable computation
MatrixBlock that = checkType(thatVal);
+ if(that instanceof CompressedMatrixBlock)
+ that = ((CompressedMatrixBlock)
that).getUncompressed("CTable " + op);
+
CTable ctable = CTable.getCTableFnObject();
double w = scalarThat2;
@@ -5465,7 +5532,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
* @param resultBlock result matrix block
* @return resultBlock
*/
- public MatrixBlock ctableSeqOperations(MatrixValue thatMatrix, double
thatScalar, MatrixBlock resultBlock) {
+ public final MatrixBlock ctableSeqOperations(MatrixValue thatMatrix,
double thatScalar, MatrixBlock resultBlock) {
return ctableSeqOperations(thatMatrix, thatScalar, resultBlock,
true);
}
@@ -5482,7 +5549,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
* @param that2Val matrix value 2
* @param resultMap table map
*/
- public void ctableOperations(Operator op, MatrixValue thatVal,
MatrixValue that2Val, CTableMap resultMap) {
+ public final void ctableOperations(Operator op, MatrixValue thatVal,
MatrixValue that2Val, CTableMap resultMap) {
ctableOperations(op, thatVal, that2Val, resultMap, null);
}
@@ -5519,7 +5586,7 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
}
}
- public MatrixBlock quaternaryOperations(QuaternaryOperator qop,
MatrixBlock um, MatrixBlock vm, MatrixBlock wm, MatrixBlock out) {
+ public final MatrixBlock quaternaryOperations(QuaternaryOperator qop,
MatrixBlock um, MatrixBlock vm, MatrixBlock wm, MatrixBlock out) {
return quaternaryOperations(qop, um, vm, wm, out, 1);
}
@@ -5815,17 +5882,17 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock, Externalizab
}
@Override
- public int compareTo(Object arg0) {
+ public final int compareTo(Object arg0) {
throw new RuntimeException("CompareTo should never be called
for matrix blocks.");
}
@Override
- public boolean equals(Object arg0) {
+ public final boolean equals(Object arg0) {
throw new RuntimeException("equals should never be called for
matrix blocks.");
}
@Override
- public int hashCode() {
+ public final int hashCode() {
throw new RuntimeException("HashCode should never be called for
matrix blocks.");
}
diff --git
a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressBase.java
b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressBase.java
index 5e1f3f5..c466e17 100644
---
a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressBase.java
+++
b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressBase.java
@@ -66,7 +66,7 @@ public abstract class CompressBase extends AutomatedTestBase {
fullDMLScriptName = SCRIPT_DIR +
"/functions/compress/compress_" + name + ".dml";
// programArgs = new String[] {"-stats", "100" ,
"-explain", "-nvargs", "A=" + input("A")};
- programArgs = new String[] {"-stats", "100" ,
"-nvargs", "A=" + input("A")};
+ programArgs = new String[] {"-stats", "100",
"-nvargs", "A=" + input("A")};
LOG.debug(runTest(null));
diff --git
a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressForce.java
b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressForce.java
index 986a799..47c8603 100644
---
a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressForce.java
+++
b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressForce.java
@@ -96,26 +96,36 @@ public class CompressForce extends CompressBase {
}
@Test
+ @Ignore
public void test_ElementWiseBinaryMultiplyOp_right_CP() {
runTest(1500, 1, 1, 1, ExecType.CP, "ewbm_right");
}
@Test
+ @Ignore
public void test_ElementWiseBinaryMultiplyOp_right_SP() {
runTest(1500, 1, 2, 1, ExecType.SPARK, "ewbm_right");
}
@Test
+ @Ignore
public void test_ElementWiseBinaryMultiplyOp_left_CP() {
runTest(1500, 1, 1, 1, ExecType.CP, "ewbm_left");
}
@Test
+ @Ignore
public void test_ElementWiseBinaryMultiplyOp_left_SP() {
runTest(1500, 1, 2, 1, ExecType.SPARK, "ewbm_left");
}
@Test
+ @Ignore
+ public void test_ElementWiseBinaryMultiplyOp_left_SP_larger() {
+ runTest(1500, 15, 2, 1, ExecType.SPARK, "ewbm_left");
+ }
+
+ @Test
public void test_ElementWiseBinaryPlusOp_CP() {
runTest(1500, 1, 0, 1, ExecType.CP, "ewbp");
}
diff --git
a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedTokenizeTest.java
b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedTokenizeTest.java
index 5af8cb6..6c2096d 100644
---
a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedTokenizeTest.java
+++
b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedTokenizeTest.java
@@ -137,12 +137,6 @@ public class FederatedTokenizeTest extends
AutomatedTestBase {
"in_S=" + input(HOME + TEST_NAME + ".json"), "rows=" +
rows, "cols=" + cols,
"out_R=" + output("S")};
runTest(null);
- try{
- Thread.sleep(10000);
- }
- catch(Exception e){
- throw new RuntimeException(e);
- }
compareResults(1e-9);
Assert.assertTrue(heavyHittersContainsString("fed_tokenize"));
TestUtils.shutdownThreads(t1, t2, t3, t4);
diff --git a/src/test/scripts/functions/compress/SystemDS-config-compress.xml
b/src/test/scripts/functions/compress/SystemDS-config-compress.xml
new file mode 100644
index 0000000..aedb884
--- /dev/null
+++ b/src/test/scripts/functions/compress/SystemDS-config-compress.xml
@@ -0,0 +1,25 @@
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+-->
+
+<root>
+ <sysds.compressed.linalg>true</sysds.compressed.linalg>
+ <sysds.defaultblocksize>1000</sysds.defaultblocksize>
+ <sysds.cp.parallel.ops>true</sysds.cp.parallel.ops>
+ <sysds.scratch>target/compress</sysds.scratch>
+</root>
\ No newline at end of file
diff --git a/src/test/scripts/functions/compress/compress_ewbm_left.dml
b/src/test/scripts/functions/compress/compress_ewbm_left.dml
index e076feb..230267b 100644
--- a/src/test/scripts/functions/compress/compress_ewbm_left.dml
+++ b/src/test/scripts/functions/compress/compress_ewbm_left.dml
@@ -20,6 +20,7 @@
#-------------------------------------------------------------
x = read($A)
+x = x + 1
b = rand(rows=nrow(x), cols=ncol(x), min=0.0, max=1.0, seed= 14);
r = b * x
print(mean(r))
diff --git a/src/test/scripts/functions/compress/compress_ewbm_right.dml
b/src/test/scripts/functions/compress/compress_ewbm_right.dml
index f9d8f99..34147a5 100644
--- a/src/test/scripts/functions/compress/compress_ewbm_right.dml
+++ b/src/test/scripts/functions/compress/compress_ewbm_right.dml
@@ -20,6 +20,7 @@
#-------------------------------------------------------------
x = read($A)
+x = x + 1
b = rand(rows=nrow(x), cols=ncol(x), min=0.0, max=1.0, seed= 14);
r = x * b
print(mean(r))
diff --git a/src/test/scripts/functions/compress/compress_ewbm_left.dml
b/src/test/scripts/functions/compress/densifying/compress_01.dml
similarity index 89%
copy from src/test/scripts/functions/compress/compress_ewbm_left.dml
copy to src/test/scripts/functions/compress/densifying/compress_01.dml
index e076feb..ea57fcc 100644
--- a/src/test/scripts/functions/compress/compress_ewbm_left.dml
+++ b/src/test/scripts/functions/compress/densifying/compress_01.dml
@@ -7,9 +7,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,11 @@
#
#-------------------------------------------------------------
-x = read($A)
-b = rand(rows=nrow(x), cols=ncol(x), min=0.0, max=1.0, seed= 14);
-r = b * x
-print(mean(r))
+x = read($1)
+
+y = compress(x)
+
+z = y + 1
+
+print(sum(z))
+
diff --git a/src/test/scripts/functions/compress/compress_ewbm_left.dml
b/src/test/scripts/functions/compress/densifying/compress_02.dml
similarity index 89%
copy from src/test/scripts/functions/compress/compress_ewbm_left.dml
copy to src/test/scripts/functions/compress/densifying/compress_02.dml
index e076feb..3084763 100644
--- a/src/test/scripts/functions/compress/compress_ewbm_left.dml
+++ b/src/test/scripts/functions/compress/densifying/compress_02.dml
@@ -7,9 +7,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,12 @@
#
#-------------------------------------------------------------
-x = read($A)
-b = rand(rows=nrow(x), cols=ncol(x), min=0.0, max=1.0, seed= 14);
-r = b * x
-print(mean(r))
+x = read($1)
+b = t(seq(0, ncol(x)-1))
+
+y = compress(x)
+
+z = y + b
+
+print(sum(z))
+
diff --git a/src/test/scripts/functions/compress/compress_ewbm_left.dml
b/src/test/scripts/functions/compress/densifying/compress_pca.dml
similarity index 89%
copy from src/test/scripts/functions/compress/compress_ewbm_left.dml
copy to src/test/scripts/functions/compress/densifying/compress_pca.dml
index e076feb..949432e 100644
--- a/src/test/scripts/functions/compress/compress_ewbm_left.dml
+++ b/src/test/scripts/functions/compress/densifying/compress_pca.dml
@@ -7,9 +7,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,11 @@
#
#-------------------------------------------------------------
-x = read($A)
-b = rand(rows=nrow(x), cols=ncol(x), min=0.0, max=1.0, seed= 14);
-r = b * x
-print(mean(r))
+x = read($1)
+
+y = compress(x)
+
+c = pca(X=y)
+
+print(sum(c))
+
diff --git a/src/test/scripts/functions/compress/compress_ewbm_left.dml
b/src/test/scripts/functions/compress/densifying/compress_scale.dml
similarity index 89%
copy from src/test/scripts/functions/compress/compress_ewbm_left.dml
copy to src/test/scripts/functions/compress/densifying/compress_scale.dml
index e076feb..2e06d0e 100644
--- a/src/test/scripts/functions/compress/compress_ewbm_left.dml
+++ b/src/test/scripts/functions/compress/densifying/compress_scale.dml
@@ -7,9 +7,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,11 @@
#
#-------------------------------------------------------------
-x = read($A)
-b = rand(rows=nrow(x), cols=ncol(x), min=0.0, max=1.0, seed= 14);
-r = b * x
-print(mean(r))
+x = read($1)
+
+y = compress(x)
+
+z = scale(y,TRUE,TRUE)
+
+print(sum(z))
+
diff --git a/src/test/scripts/functions/compress/compress_ewbm_left.dml
b/src/test/scripts/functions/compress/densifying/compress_seq.dml
similarity index 89%
copy from src/test/scripts/functions/compress/compress_ewbm_left.dml
copy to src/test/scripts/functions/compress/densifying/compress_seq.dml
index e076feb..f43ee0b 100644
--- a/src/test/scripts/functions/compress/compress_ewbm_left.dml
+++ b/src/test/scripts/functions/compress/densifying/compress_seq.dml
@@ -7,9 +7,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,12 @@
#
#-------------------------------------------------------------
-x = read($A)
-b = rand(rows=nrow(x), cols=ncol(x), min=0.0, max=1.0, seed= 14);
-r = b * x
-print(mean(r))
+x = read($1)
+b = t(seq(0, ncol(x)-1))
+
+y = compress(x)
+
+z = ((y + b) / 2) ^ 2
+
+print(sum(z))
+
diff --git a/src/test/scripts/functions/compress/compress_ewbm_left.dml
b/src/test/scripts/functions/compress/densifying/compress_submean.dml
similarity index 89%
copy from src/test/scripts/functions/compress/compress_ewbm_left.dml
copy to src/test/scripts/functions/compress/densifying/compress_submean.dml
index e076feb..90ee8c6 100644
--- a/src/test/scripts/functions/compress/compress_ewbm_left.dml
+++ b/src/test/scripts/functions/compress/densifying/compress_submean.dml
@@ -7,9 +7,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,14 @@
#
#-------------------------------------------------------------
-x = read($A)
-b = rand(rows=nrow(x), cols=ncol(x), min=0.0, max=1.0, seed= 14);
-r = b * x
-print(mean(r))
+# make one value
+x = read($1)
+
+y = compress(x)
+
+cm = colMeans(y)
+
+z = y - cm
+
+print(sum(z))
+
diff --git a/src/test/scripts/functions/compress/densifying/large.ijv
b/src/test/scripts/functions/compress/densifying/large.ijv
new file mode 100644
index 0000000..582a6d1
--- /dev/null
+++ b/src/test/scripts/functions/compress/densifying/large.ijv
@@ -0,0 +1,2 @@
+1 1 13241
+2 2 312
\ No newline at end of file
diff --git a/src/test/scripts/functions/compress/densifying/large.ijv.mtd
b/src/test/scripts/functions/compress/densifying/large.ijv.mtd
new file mode 100644
index 0000000..9d86aa5
--- /dev/null
+++ b/src/test/scripts/functions/compress/densifying/large.ijv.mtd
@@ -0,0 +1,8 @@
+{
+ "data_type": "matrix",
+ "value_type": "double",
+ "rows": 1000000,
+ "cols": 3000,
+ "nnz": 2,
+ "format": "text"
+}
\ No newline at end of file
diff --git a/src/test/scripts/functions/compress/densifying/small.ijv
b/src/test/scripts/functions/compress/densifying/small.ijv
new file mode 100644
index 0000000..582a6d1
--- /dev/null
+++ b/src/test/scripts/functions/compress/densifying/small.ijv
@@ -0,0 +1,2 @@
+1 1 13241
+2 2 312
\ No newline at end of file
diff --git a/src/test/scripts/functions/compress/densifying/small.ijv.mtd
b/src/test/scripts/functions/compress/densifying/small.ijv.mtd
new file mode 100644
index 0000000..8295a39
--- /dev/null
+++ b/src/test/scripts/functions/compress/densifying/small.ijv.mtd
@@ -0,0 +1,8 @@
+{
+ "data_type": "matrix",
+ "value_type": "double",
+ "rows": 1000,
+ "cols": 30000,
+ "nnz": 2,
+ "format": "text"
+}
\ No newline at end of file
diff --git a/src/test/scripts/functions/compress/compress_ewbm_left.dml
b/src/test/scripts/functions/compress/workload/WorkloadAnalysisLmCG.dml
similarity index 71%
copy from src/test/scripts/functions/compress/compress_ewbm_left.dml
copy to src/test/scripts/functions/compress/workload/WorkloadAnalysisLmCG.dml
index e076feb..c083425 100644
--- a/src/test/scripts/functions/compress/compress_ewbm_left.dml
+++ b/src/test/scripts/functions/compress/workload/WorkloadAnalysisLmCG.dml
@@ -7,9 +7,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,23 @@
#
#-------------------------------------------------------------
-x = read($A)
-b = rand(rows=nrow(x), cols=ncol(x), min=0.0, max=1.0, seed= 14);
-r = b * x
-print(mean(r))
+X = read($1);
+y = read($2);
+
+# Limit to one classification
+y = y == min(y)
+
+print("")
+print("LMCG")
+
+X = scale(X=X, scale=TRUE, center=TRUE);
+B = lmCG(X=X, y=y, verbose=TRUE);
+p = lmPredict(X=X, B=B, ytest=y, verbose=TRUE);
+
+classifications = (p > 0.1)
+
+acc = sum(classifications == y) / nrow(y)
+
+if(acc < 0.80)
+ stop("ERROR: to low accuracy achieved")
+print(acc)
diff --git a/src/test/scripts/functions/compress/compress_ewbm_left.dml
b/src/test/scripts/functions/compress/workload/WorkloadAnalysisLmDS.dml
similarity index 71%
copy from src/test/scripts/functions/compress/compress_ewbm_left.dml
copy to src/test/scripts/functions/compress/workload/WorkloadAnalysisLmDS.dml
index e076feb..9ed910d 100644
--- a/src/test/scripts/functions/compress/compress_ewbm_left.dml
+++ b/src/test/scripts/functions/compress/workload/WorkloadAnalysisLmDS.dml
@@ -7,9 +7,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,23 @@
#
#-------------------------------------------------------------
-x = read($A)
-b = rand(rows=nrow(x), cols=ncol(x), min=0.0, max=1.0, seed= 14);
-r = b * x
-print(mean(r))
+X = read($1);
+y = read($2);
+
+# Limit to one classification
+y = y == min(y)
+
+print("")
+print("LMDS")
+
+X = scale(X=X, scale=TRUE, center=TRUE);
+B = lmDS(X=X, y=y, verbose=TRUE);
+p = lmPredict(X=X, B=B, ytest=y, verbose=TRUE);
+
+classifications = (p > 0.1)
+
+acc = sum(classifications == y) / nrow(y)
+
+if(acc < 0.80)
+ stop("ERROR: to low accuracy achieved")
+print(acc)
diff --git
a/src/test/scripts/functions/compress/workload/WorkloadAnalysisMLogReg.dml
b/src/test/scripts/functions/compress/workload/WorkloadAnalysisMLogReg.dml
index 77b2959..12d9dd5 100644
--- a/src/test/scripts/functions/compress/workload/WorkloadAnalysisMLogReg.dml
+++ b/src/test/scripts/functions/compress/workload/WorkloadAnalysisMLogReg.dml
@@ -38,9 +38,5 @@ print(toString(C))
print(acc)
if(acc < 50){
- print("")
- print("--------------------------------------------")
- print("ERROR: Accuracy achieved is not high enough.")
- print("--------------------------------------------")
- print("")
+ stop("MLogReg Accuracy achieved is not high enough")
}
diff --git
a/src/test/scripts/functions/compress/workload/WorkloadAnalysisMLogReg.dml
b/src/test/scripts/functions/compress/workload/WorkloadAnalysisSliceFinder.dml
similarity index 71%
copy from
src/test/scripts/functions/compress/workload/WorkloadAnalysisMLogReg.dml
copy to
src/test/scripts/functions/compress/workload/WorkloadAnalysisSliceFinder.dml
index 77b2959..b287cbc 100644
--- a/src/test/scripts/functions/compress/workload/WorkloadAnalysisMLogReg.dml
+++
b/src/test/scripts/functions/compress/workload/WorkloadAnalysisSliceFinder.dml
@@ -19,28 +19,30 @@
#
#-------------------------------------------------------------
-X = read($1);
+X = read($1) + 1;
Y = read($2);
print("")
print("MLogReg")
-X = scale(X=X, scale=TRUE, center=TRUE);
-B = multiLogReg(X=X, Y=Y, verbose=FALSE, maxi=3, maxii=2);
+[X_s,s,c] = scale(X=X, scale=TRUE, center=TRUE);
+B = multiLogReg(X=X_s, Y=Y, verbose=FALSE, maxi=2, maxii=2);
+[nn, P, acc] = multiLogRegPredict(X=X_s, B=B, Y=Y)
-[nn, P, acc] = multiLogRegPredict(X=X, B=B, Y=Y)
[nn, C] = confusionMatrix(P, Y)
print("Confusion: ")
print(toString(C))
-print(acc)
-if(acc < 50){
- print("")
- print("--------------------------------------------")
- print("ERROR: Accuracy achieved is not high enough.")
- print("--------------------------------------------")
- print("")
-}
+print("")
+print("SliceFinder")
+
+e = Y == P
+
+[tk,tkc,d] = slicefinder(X=X, e=e, maxL = 2, verbose=TRUE)
+
+print("tk :\n" + toString(tk))
+print("tkc :\n" + toString(tkc))
+print("Debug matrix:\n" + toString(d))
diff --git a/src/test/scripts/functions/federated/FederatedTokenizeTest.dml
b/src/test/scripts/functions/federated/FederatedTokenizeTest.dml
index ee24a2e..e293306 100644
--- a/src/test/scripts/functions/federated/FederatedTokenizeTest.dml
+++ b/src/test/scripts/functions/federated/FederatedTokenizeTest.dml
@@ -20,8 +20,8 @@
#-------------------------------------------------------------
F1 = federated(type="frame", addresses=list($in_X1, $in_X2, $in_X3),
- ranges=list(list(0, 0), list(3, $cols), list(3, 0), list(6, $cols),
- list(6, 0), list(9, $cols)));
+ ranges=list(list(0, 0), list(2, $cols), list(2, 0), list(4, $cols),
+ list(4, 0), list(6, $cols)));
max_token = 2000;
diff --git
a/src/test/scripts/functions/federated/FederatedTokenizeTestReference.dml
b/src/test/scripts/functions/federated/FederatedTokenizeTestReference.dml
index 5522955..f6dae76 100644
--- a/src/test/scripts/functions/federated/FederatedTokenizeTestReference.dml
+++ b/src/test/scripts/functions/federated/FederatedTokenizeTestReference.dml
@@ -19,7 +19,8 @@
#
#-------------------------------------------------------------
-F = read($1);
+F = read($1, data_type="frame", format="csv", sep=",");
+F = F[2:3, 1:4];
F = rbind(F, rbind(F, F));
max_token = 2000;
