This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new b751389358 [SYSTEMDS-3827] CLA MultiCBind
b751389358 is described below
commit b751389358a65e7c4820c74706c6a04b85d7c8dd
Author: Sebastian Baunsgaard <[email protected]>
AuthorDate: Mon Feb 3 13:44:04 2025 +0100
[SYSTEMDS-3827] CLA MultiCBind
This commit adds specialized support for n way cbind in compressed space.
Closes #2208
---
.../runtime/compress/CompressedMatrixBlock.java | 6 +-
.../runtime/compress/colgroup/ColGroupConst.java | 9 +-
.../runtime/compress/colgroup/ColGroupSDCFOR.java | 22 +---
.../lib/{CLALibAppend.java => CLALibCBind.java} | 144 ++++++++++++++++++---
.../instructions/cp/MatrixAppendCPInstruction.java | 7 +-
.../sysds/runtime/matrix/data/MatrixBlock.java | 25 +++-
.../component/compress/CompressedCustomTests.java | 7 +
.../component/compress/CompressedTestBase.java | 93 ++++++++++++-
8 files changed, 263 insertions(+), 50 deletions(-)
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
index 75eadc0033..a05c076b36 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
@@ -50,8 +50,8 @@ import
org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
import org.apache.sysds.runtime.compress.colgroup.ColGroupIO;
import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed;
import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary;
-import org.apache.sysds.runtime.compress.lib.CLALibAppend;
import org.apache.sysds.runtime.compress.lib.CLALibBinaryCellOp;
+import org.apache.sysds.runtime.compress.lib.CLALibCBind;
import org.apache.sysds.runtime.compress.lib.CLALibCMOps;
import org.apache.sysds.runtime.compress.lib.CLALibCompAgg;
import org.apache.sysds.runtime.compress.lib.CLALibDecompress;
@@ -556,8 +556,8 @@ public class CompressedMatrixBlock extends MatrixBlock {
@Override
public MatrixBlock append(MatrixBlock[] that, MatrixBlock ret, boolean
cbind) {
- if(cbind && that.length == 1)
- return CLALibAppend.append(this, that[0],
InfrastructureAnalyzer.getLocalParallelism());
+ if(cbind)
+ return CLALibCBind.cbind(this, that,
InfrastructureAnalyzer.getLocalParallelism());
else {
MatrixBlock left = getUncompressed("append list or
r-bind not supported in compressed");
MatrixBlock[] thatUC = new MatrixBlock[that.length];
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
index d3ca10445c..a493b14f04 100644
---
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
+++
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
@@ -655,7 +655,7 @@ public class ColGroupConst extends ADictBasedColGroup
implements IContainDefault
public double getSparsity() {
return 1.0;
}
-
+
@Override
protected void sparseSelection(MatrixBlock selection, P[] points,
MatrixBlock ret, int rl, int ru) {
throw new NotImplementedException();
@@ -710,12 +710,10 @@ public class ColGroupConst extends ADictBasedColGroup
implements IContainDefault
public AColGroup combineWithSameIndex(int nRow, int nCol, AColGroup
right) {
if(!(right instanceof ColGroupConst))
return super.combineWithSameIndex(nRow, nCol, right);
-
final IColIndex combIndex =
_colIndexes.combine(right.getColIndices().shift(nCol));
final IDictionary b = ((ColGroupConst) right).getDictionary();
final IDictionary combined =
DictionaryFactory.cBindDictionaries(_dict, b, this.getNumCols(),
right.getNumCols());
return create(combIndex, combined);
-
}
@Override
@@ -737,10 +735,11 @@ public class ColGroupConst extends ADictBasedColGroup
implements IContainDefault
for(int i = 0; i < right.size(); i++) {
AColGroup g = right.get(i);
- if(!(g instanceof ColGroupConst) || !(g instanceof
ColGroupEmpty)) {
+ if(!(g instanceof ColGroupConst) && !(g instanceof
ColGroupEmpty)) {
return super.combineWithSameIndex(nRow, nCol,
right);
}
}
+
IColIndex combinedIndex = _colIndexes;
int i = 0;
for(AColGroup g : right) {
@@ -751,7 +750,7 @@ public class ColGroupConst extends ADictBasedColGroup
implements IContainDefault
return create(combinedIndex, combined);
}
-
+
@Override
protected boolean allowShallowIdentityRightMult() {
return true;
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
index de14c293df..4c4b2e20a5 100644
---
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
+++
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
@@ -549,25 +549,12 @@ public class ColGroupSDCFOR extends ASDC implements
IMapToDataGroup, IFrameOfRef
final IColIndex combinedColIndex = combineColIndexes(nCol,
right);
final double[] combinedDefaultTuple =
IContainDefaultTuple.combineDefaultTuples(_reference, right);
- // return new ColGroupDDC(combinedColIndex, combined, _data,
getCachedCounts());
- return new ColGroupSDC(combinedColIndex, this.getNumRows(),
combined, combinedDefaultTuple, _indexes, _data,
- getCachedCounts());
+ return new ColGroupSDCFOR(combinedColIndex, this.getNumRows(),
combined, _indexes, _data, getCachedCounts(),
+ combinedDefaultTuple);
}
@Override
public AColGroupCompressed combineWithSameIndex(int nRow, int nCol,
AColGroup right) {
- // if(right instanceof ColGroupSDCZeros){
- // ColGroupSDCZeros rightSDC = ((ColGroupSDCZeros) right);
- // IDictionary b = rightSDC.getDictionary();
- // IDictionary combined =
DictionaryFactory.cBindDictionaries(_dict, b, this.getNumCols(),
right.getNumCols());
- // IColIndex combinedColIndex =
_colIndexes.combine(right.getColIndices().shift(nCol));
- // double[] combinedDefaultTuple = new double[_reference.length
+ right.getNumCols()];
- // System.arraycopy(_reference, 0, combinedDefaultTuple, 0,
_reference.length);
-
- // return new ColGroupSDC(combinedColIndex, this.getNumRows(),
combined, combinedDefaultTuple, _indexes, _data,
- // getCachedCounts());
- // }
- // else{
ColGroupSDCFOR rightSDC = ((ColGroupSDCFOR) right);
IDictionary b = rightSDC.getDictionary();
IDictionary combined =
DictionaryFactory.cBindDictionaries(_dict, b, this.getNumCols(),
right.getNumCols());
@@ -576,9 +563,8 @@ public class ColGroupSDCFOR extends ASDC implements
IMapToDataGroup, IFrameOfRef
System.arraycopy(_reference, 0, combinedDefaultTuple, 0,
_reference.length);
System.arraycopy(rightSDC._reference, 0, combinedDefaultTuple,
_reference.length, rightSDC._reference.length);
- return new ColGroupSDC(combinedColIndex, this.getNumRows(),
combined, combinedDefaultTuple, _indexes, _data,
- getCachedCounts());
- // }
+ return new ColGroupSDCFOR(combinedColIndex, this.getNumRows(),
combined, _indexes, _data, getCachedCounts(),
+ combinedDefaultTuple);
}
@Override
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCBind.java
similarity index 56%
rename from
src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
rename to src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCBind.java
index cedf98494c..49533e4bcc 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCBind.java
@@ -21,27 +21,60 @@ package org.apache.sysds.runtime.compress.lib;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
+import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.colgroup.AColGroup;
import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed;
import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.runtime.util.CommonThreadPool;
-public final class CLALibAppend {
+public final class CLALibCBind {
- private CLALibAppend(){
+ private CLALibCBind() {
// private constructor.
}
- private static final Log LOG =
LogFactory.getLog(CLALibAppend.class.getName());
+ private static final Log LOG =
LogFactory.getLog(CLALibCBind.class.getName());
- public static MatrixBlock append(MatrixBlock left, MatrixBlock right,
int k) {
+ public static MatrixBlock cbind(MatrixBlock left, MatrixBlock[] right,
int k) {
+ try {
+
+ if(right.length == 1) {
+ return cbind(left, right[0], k);
+ }
+ else {
+ boolean allCompressed = true;
+ for(int i = 0; i < right.length &&
allCompressed; i++)
+ allCompressed = right[i] instanceof
CompressedMatrixBlock;
+ if(allCompressed)
+ return
cbindAllCompressed((CompressedMatrixBlock) left, right, k);
+ else
+ return cbindAllNormalCompressed(left,
right, k);
+ }
+ }
+ catch(Exception e) {
+ throw new DMLCompressionException("Failed to Cbind with
compressed input", e);
+ }
+ }
+
+ private static MatrixBlock cbindAllNormalCompressed(MatrixBlock left,
MatrixBlock[] right, int k) {
+ for(int i = 0; i < right.length; i++) {
+ left = cbind(left, right[i], k);
+ }
+ return left;
+ }
+
+ public static MatrixBlock cbind(MatrixBlock left, MatrixBlock right,
int k) {
final int m = left.getNumRows();
final int n = left.getNumColumns() + right.getNumColumns();
@@ -66,6 +99,9 @@ public final class CLALibAppend {
final double spar = (left.getNonZeros() +
right.getNonZeros()) / ((double) m * n);
final double estSizeUncompressed =
MatrixBlock.estimateSizeInMemory(m, n, spar);
final double estSizeCompressed = left.getInMemorySize()
+ right.getInMemorySize();
+ // if(isAligned((CompressedMatrixBlock) left,
(CompressedMatrixBlock) right))
+ // return combineCompressed((CompressedMatrixBlock)
left, (CompressedMatrixBlock) right);
+ // else
if(estSizeUncompressed < estSizeCompressed)
return uc(left).append(uc(right), null);
else if(left instanceof CompressedMatrixBlock)
@@ -73,8 +109,86 @@ public final class CLALibAppend {
else
return appendLeftUncompressed(left,
(CompressedMatrixBlock) right, m, n);
}
+ if(isAligned((CompressedMatrixBlock) left,
(CompressedMatrixBlock) right))
+ return combineCompressed((CompressedMatrixBlock) left,
(CompressedMatrixBlock) right);
+ else
+ return append((CompressedMatrixBlock) left,
(CompressedMatrixBlock) right, m, n);
+ }
+
+ private static MatrixBlock cbindAllCompressed(CompressedMatrixBlock
left, MatrixBlock[] right, int k)
+ throws InterruptedException, ExecutionException {
+
+ final int nCol = left.getNumColumns();
+ for(int i = 0; i < right.length; i++) {
+ CompressedMatrixBlock rightCM =
((CompressedMatrixBlock) right[i]);
+ if(nCol != right[i].getNumColumns() || !isAligned(left,
rightCM))
+ return cbindAllNormalCompressed(left, right, k);
+ }
+ return cbindAllCompressedAligned(left, right, k);
+
+ }
+
+ private static boolean isAligned(CompressedMatrixBlock left,
CompressedMatrixBlock right) {
+ final List<AColGroup> gl = left.getColGroups();
+ for(int j = 0; j < gl.size(); j++) {
+ final AColGroup glj = gl.get(j);
+ final int aColumnInGroup = glj.getColIndices().get(0);
+ final AColGroup grj =
right.getColGroupForColumn(aColumnInGroup);
+
+ if(!glj.sameIndexStructure(grj) || glj.getNumCols() !=
grj.getNumCols())
+ return false;
+
+ }
+ return true;
+ }
+
+ private static CompressedMatrixBlock
combineCompressed(CompressedMatrixBlock left, CompressedMatrixBlock right) {
+ final List<AColGroup> gl = left.getColGroups();
+ final List<AColGroup> retCG = new ArrayList<>(gl.size());
+ for(int j = 0; j < gl.size(); j++) {
+ AColGroup glj = gl.get(j);
+ int aColumnInGroup = glj.getColIndices().get(0);
+ AColGroup grj =
right.getColGroupForColumn(aColumnInGroup);
+ // parallel combine...
+ retCG.add(glj.combineWithSameIndex(left.getNumRows(),
left.getNumColumns(), grj));
+ }
+ return new CompressedMatrixBlock(left.getNumRows(),
left.getNumColumns() + right.getNumColumns(),
+ left.getNonZeros() + right.getNonZeros(), false, retCG);
+ }
+
+ private static CompressedMatrixBlock
cbindAllCompressedAligned(CompressedMatrixBlock left, MatrixBlock[] right,
+ final int k) throws InterruptedException, ExecutionException {
+
+ final ExecutorService pool = CommonThreadPool.get(k);
+ try {
+ final List<AColGroup> gl = left.getColGroups();
+ final List<Future<AColGroup>> tasks = new ArrayList<>();
+ final int nCol = left.getNumColumns();
+ final int nRow = left.getNumRows();
+ for(int i = 0; i < gl.size(); i++) {
+ final AColGroup gli = gl.get(i);
+ tasks.add(pool.submit(() -> {
+ List<AColGroup> combines = new
ArrayList<>();
+ final int cId =
gli.getColIndices().get(0);
+ for(int j = 0; j < right.length; j++) {
+
combines.add(((CompressedMatrixBlock) right[j]).getColGroupForColumn(cId));
+ }
+ return gli.combineWithSameIndex(nRow,
nCol, combines);
+ }));
+ }
+
+ final List<AColGroup> retCG = new
ArrayList<>(gl.size());
+ for(Future<AColGroup> t : tasks)
+ retCG.add(t.get());
+
+ int totalCol = nCol + right.length * nCol;
+
+ return new CompressedMatrixBlock(left.getNumRows(),
totalCol, -1, false, retCG);
+ }
+ finally {
+ pool.shutdown();
+ }
- return append((CompressedMatrixBlock) left,
(CompressedMatrixBlock) right, m, n);
}
private static MatrixBlock appendLeftUncompressed(MatrixBlock left,
CompressedMatrixBlock right, final int m,
@@ -123,17 +237,17 @@ public final class CLALibAppend {
ret.setNonZeros(left.getNonZeros() + right.getNonZeros());
ret.setOverlapping(left.isOverlapping() ||
right.isOverlapping());
- final double compressedSize = ret.getInMemorySize();
- final double uncompressedSize =
MatrixBlock.estimateSizeInMemory(m, n, ret.getSparsity());
+ // final double compressedSize = ret.getInMemorySize();
+ // final double uncompressedSize =
MatrixBlock.estimateSizeInMemory(m, n, ret.getSparsity());
- if(compressedSize < uncompressedSize)
- return ret;
- else {
- final double ratio = uncompressedSize / compressedSize;
- String message = String.format("Decompressing c bind
matrix because it had to small compression ratio: %2.3f",
- ratio);
- return ret.getUncompressed(message);
- }
+ // if(compressedSize < uncompressedSize)
+ return ret;
+ // else {
+ // final double ratio = uncompressedSize / compressedSize;
+ // String message = String.format("Decompressing c bind matrix
because it had to small compression ratio: %2.3f",
+ // ratio);
+ // return ret.getUncompressed(message);
+ // }
}
private static MatrixBlock appendRightEmpty(CompressedMatrixBlock left,
MatrixBlock right, int m, int n) {
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/cp/MatrixAppendCPInstruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/cp/MatrixAppendCPInstruction.java
index 9027d4514a..1b151e0832 100644
---
a/src/main/java/org/apache/sysds/runtime/instructions/cp/MatrixAppendCPInstruction.java
+++
b/src/main/java/org/apache/sysds/runtime/instructions/cp/MatrixAppendCPInstruction.java
@@ -22,7 +22,7 @@ package org.apache.sysds.runtime.instructions.cp;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
-import org.apache.sysds.runtime.compress.lib.CLALibAppend;
+import org.apache.sysds.runtime.compress.lib.CLALibCBind;
import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysds.runtime.lineage.LineageItem;
import org.apache.sysds.runtime.lineage.LineageItemUtils;
@@ -46,8 +46,9 @@ public final class MatrixAppendCPInstruction extends
AppendCPInstruction {
validateInput(matBlock1, matBlock2);
MatrixBlock ret;
- if(matBlock1 instanceof CompressedMatrixBlock || matBlock2
instanceof CompressedMatrixBlock)
- ret = CLALibAppend.append(matBlock1, matBlock2,
InfrastructureAnalyzer.getLocalParallelism());
+ if(_type == AppendType.CBIND &&
+ (matBlock1 instanceof CompressedMatrixBlock ||
matBlock2 instanceof CompressedMatrixBlock))
+ ret = CLALibCBind.cbind(matBlock1, matBlock2,
InfrastructureAnalyzer.getLocalParallelism());
else
ret = matBlock1.append(matBlock2, new MatrixBlock(),
_type == AppendType.CBIND);
diff --git
a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
index 43610820fa..c9086778f0 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
@@ -56,6 +56,7 @@ import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.lib.CLALibAggTernaryOp;
+import org.apache.sysds.runtime.compress.lib.CLALibCBind;
import org.apache.sysds.runtime.compress.lib.CLALibMerge;
import org.apache.sysds.runtime.compress.lib.CLALibTernaryOp;
import org.apache.sysds.runtime.controlprogram.caching.CacheBlock;
@@ -3654,10 +3655,19 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock<MatrixBlock>,
return append(that, ret, true); //default cbind
}
- public static MatrixBlock append(List<MatrixBlock> that,MatrixBlock
ret, boolean cbind, int k ){
- MatrixBlock[] th = new MatrixBlock[that.size() -1];
- for(int i = 0; i < that.size() -1; i++)
- th[i] = that.get(i+1);
+ /**
+ * Append that list of matrixblocks to this.
+ *
+ * @param that That list.
+ * @param ret The output block
+ * @param cbind If the blocks a appended cbind
+ * @param k the parallelization degree
+ * @return the appended matrix.
+ */
+ public static MatrixBlock append(List<MatrixBlock> that, MatrixBlock
ret, boolean cbind, int k) {
+ MatrixBlock[] th = new MatrixBlock[that.size() - 1];
+ for(int i = 0; i < that.size() - 1; i++)
+ th[i] = that.get(i + 1);
return that.get(0).append(th, ret, cbind);
}
@@ -3716,6 +3726,13 @@ public class MatrixBlock extends MatrixValue implements
CacheBlock<MatrixBlock>,
public MatrixBlock append(MatrixBlock[] that, MatrixBlock result,
boolean cbind) {
checkDimensionsForAppend(that, cbind);
+ for(int k = 0; k < that.length; k++)
+ if( that[k] instanceof CompressedMatrixBlock){
+ if(that.length == 1 && cbind)
+ return CLALibCBind.cbind(this, that[0],
1);
+ that[k] =
CompressedMatrixBlock.getUncompressed(that[k], "Append N");
+ }
+
final int m = cbind ? rlen : combinedRows(that);
final int n = cbind ? combinedCols(that) : clen;
final long nnz = calculateCombinedNNz(that);
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/CompressedCustomTests.java
b/src/test/java/org/apache/sysds/test/component/compress/CompressedCustomTests.java
index 13ccb8924c..32d62fb16c 100644
---
a/src/test/java/org/apache/sysds/test/component/compress/CompressedCustomTests.java
+++
b/src/test/java/org/apache/sysds/test/component/compress/CompressedCustomTests.java
@@ -37,6 +37,7 @@ import org.apache.sysds.runtime.compress.cost.ACostEstimate;
import org.apache.sysds.runtime.compress.cost.CostEstimatorBuilder;
import org.apache.sysds.runtime.compress.cost.CostEstimatorFactory;
import org.apache.sysds.runtime.compress.cost.InstructionTypeCounter;
+import org.apache.sysds.runtime.compress.lib.CLALibCBind;
import org.apache.sysds.runtime.compress.workload.WTreeRoot;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.test.TestUtils;
@@ -395,4 +396,10 @@ public class CompressedCustomTests {
MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m1).getLeft();
TestUtils.compareMatricesBitAvgDistance(m1, m2, 0, 0, "no");
}
+
+
+ @Test(expected = Exception.class)
+ public void cbindWithError(){
+ CLALibCBind.cbind(null, new MatrixBlock[]{null}, 0);
+ }
}
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java
b/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java
index de968a483f..507a2fc663 100644
---
a/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java
+++
b/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java
@@ -19,6 +19,7 @@
package org.apache.sysds.test.component.compress;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.util.ArrayList;
@@ -58,7 +59,7 @@ import
org.apache.sysds.runtime.compress.cost.InstructionTypeCounter;
import org.apache.sysds.runtime.compress.estim.ComEstFactory;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfo;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
-import org.apache.sysds.runtime.compress.lib.CLALibAppend;
+import org.apache.sysds.runtime.compress.lib.CLALibCBind;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.functionobjects.Builtin.BuiltinCode;
import org.apache.sysds.runtime.functionobjects.Divide;
@@ -255,7 +256,7 @@ public abstract class CompressedTestBase extends TestBase {
case C_BIND_SELF:
if(cmb instanceof
CompressedMatrixBlock) {
CompressedMatrixBlock cmbc =
(CompressedMatrixBlock) cmb;
- cmb = CLALibAppend.append(cmbc,
cmbc, _k);
+ cmb = CLALibCBind.cbind(cmbc,
cmbc, _k);
mb = mb.append(mb, new
MatrixBlock());
cols *= 2;
}
@@ -1078,6 +1079,45 @@ public abstract class CompressedTestBase extends
TestBase {
compareResultMatrices(ret1, ret2, 1);
}
+ @Test
+ public void appendMultiple() {
+ try {
+
+ if(!(cmb instanceof CompressedMatrixBlock))
+ return;
+ if(vectorRows == null)
+ vectorRows =
CompressibleInputGenerator.getInput(rows, 1, CompressionType.OLE, 5, 5, -5,
1.0, 3);
+ MatrixBlock ap =
CompressedMatrixBlockFactory.compress(vectorRows).getLeft();
+ MatrixBlock ret1 = mb.append(new MatrixBlock[] {ap,
ap}, new MatrixBlock(), true);
+ MatrixBlock ret2 = cmb.append(new MatrixBlock[] {ap,
ap}, new MatrixBlock(), true);
+ compareResultMatrices(ret1, ret2, 1);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void appendMultipleNotAllCompressed() {
+ try {
+
+ if(!(cmb instanceof CompressedMatrixBlock))
+ return;
+ if(vectorRows == null)
+ vectorRows =
CompressibleInputGenerator.getInput(rows, 1, CompressionType.OLE, 5, 5, -5,
1.0, 3);
+ MatrixBlock ap = vectorRows;
+ MatrixBlock apc =
CompressedMatrixBlockFactory.compress(vectorRows).getLeft();
+ MatrixBlock ret1 = mb.append(new MatrixBlock[] {ap,
apc}, new MatrixBlock(), true);
+ MatrixBlock ret2 = cmb.append(new MatrixBlock[] {ap,
apc}, new MatrixBlock(), true);
+ compareResultMatrices(ret1, ret2, 1);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
@Test
public void appendCBindTrue() {
if(!(cmb instanceof CompressedMatrixBlock) || rows * cols >
10000)
@@ -1088,6 +1128,55 @@ public abstract class CompressedTestBase extends
TestBase {
compareResultMatrices(ret1, ret2, 1);
}
+ @Test
+ public void appendCBindTrueEmpty() {
+ if(!(cmb instanceof CompressedMatrixBlock) || rows * cols >
10000)
+ return;
+ MatrixBlock ap = new MatrixBlock(mb.getNumRows(), 1, 0.0);
+ assertTrue(ap.isEmpty());
+ MatrixBlock ret1 = mb.append(ap, new MatrixBlock(), true);
+ MatrixBlock ret2 = cmb.append(ap, new MatrixBlock(), true);
+ compareResultMatrices(ret1, ret2, 1);
+ }
+
+ @Test
+ public void appendCBindTrueEmptyInverse() {
+ if(!(cmb instanceof CompressedMatrixBlock) || rows * cols >
10000)
+ return;
+ MatrixBlock ap = new MatrixBlock(mb.getNumRows(), 1, 0.0);
+ assertTrue(ap.isEmpty());
+ MatrixBlock ret1 = ap.append(mb, new MatrixBlock(), true);
+ MatrixBlock ret2 = ap.append(cmb, new MatrixBlock(), true);
+ compareResultMatrices(ret1, ret2, 1);
+ }
+
+ @Test
+ public void appendCBindTrueConstInverse() {
+ if(!(cmb instanceof CompressedMatrixBlock) || rows * cols >
10000)
+ return;
+ MatrixBlock ap = new MatrixBlock(mb.getNumRows(), 1, 2.0);
+ assertTrue(!ap.isEmpty());
+ MatrixBlock ret1 = ap.append(mb, new MatrixBlock(), true);
+ MatrixBlock ret2 = ap.append(cmb, new MatrixBlock(), true);
+ compareResultMatrices(ret1, ret2, 1);
+ }
+
+ @Test
+ public void appendCBindAlignedSelfMultiple() {
+ try {
+
+ if(!(cmb instanceof CompressedMatrixBlock) || rows *
cols > 10000)
+ return;
+ MatrixBlock ret1 = mb.append(new MatrixBlock[] {mb,
mb}, new MatrixBlock(), true);
+ MatrixBlock ret2 = cmb.append(new MatrixBlock[] {cmb,
cmb}, new MatrixBlock(), true);
+ compareResultMatrices(ret1, ret2, 1);
+ }
+ catch(AssertionError e) {
+ e.printStackTrace();
+ fail("failed Cbind: " + cmb.toString() );
+ }
+ }
+
@Test
public void appendCBindFalse() {
if(!(cmb instanceof CompressedMatrixBlock) || rows * cols >
10000)
