This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new e11d20a59a [SYSTEMDS-3526] Transpose dense to sparse
e11d20a59a is described below
commit e11d20a59a0a2b8cb09717243c179803429c1768
Author: baunsgaard <[email protected]>
AuthorDate: Fri Apr 21 12:33:30 2023 +0200
[SYSTEMDS-3526] Transpose dense to sparse
Before: single thread transpose dense to sparse dominated:
census_enc_16k-kmeans+-claWorkloadb16 -- dams-so001
r' 15.085 112
I removed an indirection of allocation via append on MCSR and
managed the sparse vectors directly:
r' 9.334 112
And finally parallelized:
r' 4.454 112
Furthermore this commit extends the component transpose tests
to verify behavior in various parameterized scenarios.
Closes #1809
---
.../colgroup/dictionary/MatrixBlockDictionary.java | 6 +-
.../compress/readers/ReaderColumnSelection.java | 17 +-
.../readers/ReaderColumnSelectionEmpty.java | 36 ++++
.../sysds/runtime/matrix/data/LibMatrixReorg.java | 187 ++++++++++++++-------
.../component/compress/readers/ReadersTest.java | 2 +-
.../readers/ReadersTestCompareReaders.java | 49 ++++--
.../test/component/matrix/TransposeCSRTest.java | 1 -
.../test/component/matrix/TransposeTwiceTest.java | 129 ++++++++++++++
8 files changed, 348 insertions(+), 79 deletions(-)
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
index 7126f8e532..a1b11eaf29 100644
---
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
+++
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
@@ -318,8 +318,10 @@ public class MatrixBlockDictionary extends ADictionary {
public void aggregateCols(double[] c, Builtin fn, IColIndex colIndexes)
{
if(_data.isInSparseFormat()) {
MatrixBlock t = LibMatrixReorg.transpose(_data);
- if(!t.isInSparseFormat()) // highly unlikely.
- throw new NotImplementedException("Not
implemented aggregate Cols on dense transposed dict.");
+ if(!t.isInSparseFormat()){
+ LOG.warn("Transpose for aggregating of
columns");
+ t.denseToSparse(true);
+ }
SparseBlock sbt = t.getSparseBlock();
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/readers/ReaderColumnSelection.java
b/src/main/java/org/apache/sysds/runtime/compress/readers/ReaderColumnSelection.java
index 019e1e2380..8f5661ee93 100644
---
a/src/main/java/org/apache/sysds/runtime/compress/readers/ReaderColumnSelection.java
+++
b/src/main/java/org/apache/sysds/runtime/compress/readers/ReaderColumnSelection.java
@@ -36,6 +36,7 @@ public abstract class ReaderColumnSelection {
protected final IColIndex _colIndexes;
protected final DblArray reusableReturn;
protected final double[] reusableArr;
+ /** The row index to stop the reading at */
protected final int _ru;
/** rl is used as a pointer to current row */
@@ -76,11 +77,14 @@ public abstract class ReaderColumnSelection {
return createReader(rawBlock, colIndices, transposed, rl, ru);
}
-
- public static ReaderColumnSelection createReader(MatrixBlock rawBlock,
IColIndex colIndices, boolean transposed, int rl,
- int ru) {
+ public static ReaderColumnSelection createReader(MatrixBlock rawBlock,
IColIndex colIndices, boolean transposed,
+ int rl, int ru) {
checkInput(rawBlock, colIndices, rl, ru);
rl = rl - 1;
+ if(rawBlock.isEmpty()) {
+ LOG.warn("It is likely an error occurred when reading
an empty block. But we do support it!");
+ return new ReaderColumnSelectionEmpty(rawBlock,
colIndices, rl, ru, transposed);
+ }
if(transposed) {
if(rawBlock.isInSparseFormat())
@@ -99,15 +103,16 @@ public abstract class ReaderColumnSelection {
private static void checkInput(final MatrixBlock rawBlock, final
IColIndex colIndices, final int rl, final int ru) {
if(colIndices.size() <= 1)
- throw new DMLCompressionException("Column selection
reader should not be done on single column groups: " + colIndices);
+ throw new DMLCompressionException(
+ "Column selection reader should not be done on
single column groups: " + colIndices);
else if(rawBlock.getSparseBlock() == null &&
rawBlock.getDenseBlock() == null)
throw new DMLCompressionException("Input Block was
null");
else if(rl >= ru)
throw new DMLCompressionException("Invalid inverse
range for reader " + rl + " to " + ru);
}
- protected void warnNaN(){
- if(!nanEncountered){
+ protected void warnNaN() {
+ if(!nanEncountered) {
LOG.warn("NaN value encountered, replaced by 0 in
compression, since nan is not supported");
nanEncountered = true;
}
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/readers/ReaderColumnSelectionEmpty.java
b/src/main/java/org/apache/sysds/runtime/compress/readers/ReaderColumnSelectionEmpty.java
new file mode 100644
index 0000000000..c428df77d6
--- /dev/null
+++
b/src/main/java/org/apache/sysds/runtime/compress/readers/ReaderColumnSelectionEmpty.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.readers;
+
+import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
+import org.apache.sysds.runtime.compress.utils.DblArray;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+
+public class ReaderColumnSelectionEmpty extends ReaderColumnSelection {
+
+ protected ReaderColumnSelectionEmpty(MatrixBlock data, IColIndex
colIndices, int rl, int ru, boolean transposed) {
+ super(colIndices, rl, Math.min(ru, 1 - (transposed ?
data.getNumColumns() : data.getNumRows())));
+ }
+
+ protected DblArray getNextRow() {
+ _rl = _ru;
+ return null;
+ }
+}
diff --git
a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java
b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java
index 2651327957..49bdc7145f 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java
@@ -35,6 +35,8 @@ import java.util.concurrent.Future;
import java.util.stream.Collectors;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.compress.DMLCompressionException;
@@ -43,6 +45,7 @@ import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.DenseBlockFactory;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.data.SparseBlockCSR;
+import org.apache.sysds.runtime.data.SparseBlockMCSR;
import org.apache.sysds.runtime.data.SparseRowVector;
import org.apache.sysds.runtime.functionobjects.DiagIndex;
import org.apache.sysds.runtime.functionobjects.RevIndex;
@@ -71,7 +74,7 @@ import org.apache.sysds.runtime.util.UtilFunctions;
*/
public class LibMatrixReorg {
- // private static final Log LOG =
LogFactory.getLog(LibMatrixReorg.class.getName());
+ protected static final Log LOG =
LogFactory.getLog(LibMatrixReorg.class.getName());
//minimum number of elements for multi-threaded execution
public static long PAR_NUMCELL_THRESHOLD = 1024*1024; //1M
@@ -145,8 +148,12 @@ public class LibMatrixReorg {
}
}
- public static MatrixBlock transpose(MatrixBlock in){
- return transpose(in, new MatrixBlock(in.getNumColumns(),
in.getNumRows(), in.isInSparseFormat()));
+ public static MatrixBlock transpose(MatrixBlock in) {
+ final int clen = in.getNumColumns();
+ final int rlen = in.getNumRows();
+ final long nnz = in.getNonZeros();
+ final boolean sparseOut =
MatrixBlock.evalSparseFormatInMemory(clen, rlen, nnz, true);
+ return transpose(in, new MatrixBlock(clen, rlen, sparseOut));
}
public static MatrixBlock transpose( MatrixBlock in, MatrixBlock out ) {
@@ -195,28 +202,43 @@ public class LibMatrixReorg {
return out;
}
- public static MatrixBlock transpose(MatrixBlock in, int k){
- return transpose(in, new MatrixBlock(in.getNumColumns(),
in.getNumRows(), in.isInSparseFormat()), k);
+ public static MatrixBlock transpose(MatrixBlock in, int k) {
+ final int clen = in.getNumColumns();
+ final int rlen = in.getNumRows();
+ final long nnz = in.getNonZeros();
+ final boolean sparseOut =
MatrixBlock.evalSparseFormatInMemory(clen, rlen, nnz, true);
+ return transpose(in, new MatrixBlock(clen, rlen, sparseOut), k);
}
-
+
public static MatrixBlock transpose( MatrixBlock in, MatrixBlock out,
int k ) {
return transpose(in, out, k, false);
}
public static MatrixBlock transpose(MatrixBlock in, MatrixBlock out,
int k, boolean allowCSR) {
// redirect small or special cases to sequential execution
- if(in.isEmptyBlock(false) || ((long) in.rlen * (long) in.clen <
PAR_NUMCELL_THRESHOLD) || k == 1
- || (SHALLOW_COPY_REORG && !in.sparse && !out.sparse &&
(in.rlen == 1 || in.clen == 1))
- || (in.sparse && !out.sparse && in.rlen == 1) ||
(!in.sparse && out.sparse && in.rlen == 1)
- || (in.sparse && out.sparse && in.nonZeros <
Math.max(in.rlen, in.clen)) //ultra-sparse
- || (!in.sparse && out.sparse) ) {
+ if(in.isEmptyBlock(false) //
+ || ((long) in.rlen * (long) in.clen <
PAR_NUMCELL_THRESHOLD) //
+ || k <= 1 //
+ || (SHALLOW_COPY_REORG && !in.sparse && !out.sparse &&
(in.rlen == 1 || in.clen == 1)) //
+ || (in.sparse && !out.sparse && in.rlen == 1) //
+ || (!in.sparse && out.sparse && in.rlen == 1) //
+ || (in.sparse && out.sparse && in.nonZeros <
Math.max(in.rlen, in.clen)) // ultra-sparse
+ ) {
return transpose(in, out);
}
// set meta data and allocate output arrays (if required)
out.nonZeros = in.nonZeros;
+
+ if(!in.sparse && out.sparse){
+ // special case dense to sparse is different than
others because appending to sparse rows.
+ transposeDenseToSparse(in, out, k);
+ return out;
+ }
+
+ // Timing time = new Timing(true);
+
// CSR is only allowed in the transposed output if the number
of non zeros is counted in the columns
allowCSR = allowCSR && out.nonZeros < (long) Integer.MAX_VALUE
&& in.clen <= 4096;
- // Timing time = new Timing(true);
if(out.sparse && allowCSR) {
int size = (int) out.nonZeros;
@@ -229,7 +251,7 @@ public class LibMatrixReorg {
// core multi-threaded transpose
try {
- ExecutorService pool = CommonThreadPool.get(k);
+ final ExecutorService pool = CommonThreadPool.get(k);
// pre-processing (compute nnz per column once for
sparse)
int[] cnt = null;
// filter matrices with many columns since the
CountNnzTask would return
@@ -272,17 +294,13 @@ public class LibMatrixReorg {
return out;
}
- public static int[] countNNZColumns(MatrixBlock in, int k,
ExecutorService pool) {
- try {
- int[] cnt = null;
- List<Future<int[]>> rtasks =
countNNZColumnsFuture(in,k,pool);
- for(Future<int[]> rtask : rtasks)
- cnt = mergeNnzCounts(cnt, rtask.get());
- return cnt;
- }
- catch(InterruptedException | ExecutionException e) {
- throw new DMLRuntimeException(e);
- }
+ public static int[] countNNZColumns(MatrixBlock in, int k,
ExecutorService pool)
+ throws InterruptedException, ExecutionException {
+ int[] cnt = null;
+ List<Future<int[]>> rtasks = countNNZColumnsFuture(in, k, pool);
+ for(Future<int[]> rtask : rtasks)
+ cnt = mergeNnzCounts(cnt, rtask.get());
+ return cnt;
}
public static List<Future<int[]>> countNNZColumnsFuture(MatrixBlock in,
int k, ExecutorService pool) throws InterruptedException {
@@ -300,7 +318,7 @@ public class LibMatrixReorg {
out = new MatrixBlock(in.getNumColumns(),
in.getNumRows(), true);
else if(in.isInSparseFormat()) {
// If input is sparse use default implementation and
allocate a new matrix.
- out = transpose(in, new MatrixBlock(in.getNumColumns(),
in.getNumRows(), true), k);
+ out = transpose(in, new MatrixBlock(in.getNumColumns(),
in.getNumRows(), true), k, true);
}
else {
transposeInPlaceDense(in, k);
@@ -918,49 +936,104 @@ public class LibMatrixReorg {
}
}
- private static void transposeDenseToSparse(MatrixBlock in, MatrixBlock
out)
- {
- //NOTE: called only in sequential execution
-
+ private static void transposeDenseToSparse(MatrixBlock in, MatrixBlock
out){
+ transposeDenseToSparse(in, out, 1);
+ }
+
+ private static void transposeDenseToSparse(MatrixBlock in, MatrixBlock
out, int k){
+ if( out.rlen == 1 )
+ transposeDenseToSparseVV(in,out);
+ else
+ transposeDenseToSparseMM(in, out, k);
+ }
+
+ private static void transposeDenseToSparseVV(MatrixBlock in,
MatrixBlock out){
+ final int m = in.rlen;
+ final DenseBlock a = in.getDenseBlock();
+ out.allocateSparseRowsBlock(false);
+ final SparseBlock c = out.getSparseBlock();
+ c.set(0, new SparseRowVector((int)in.nonZeros, a.valuesAt(0),
m), false);
+ }
+
+ private static void transposeDenseToSparseMM(MatrixBlock in,
MatrixBlock out, int k){
final int m = in.rlen;
final int n = in.clen;
final int m2 = out.rlen;
final int n2 = out.clen;
final int ennz2 = (int) (in.nonZeros/m2);
-
- DenseBlock a = in.getDenseBlock();
- SparseBlock c = out.getSparseBlock();
-
- if( out.rlen == 1 ) //VECTOR-VECTOR
- {
- //allocate row once by nnz, copy non-zeros
- c.set(0, new SparseRowVector((int)in.nonZeros,
a.valuesAt(0), m), false);
+ final DenseBlock a = in.getDenseBlock();
+
+ final SparseRowVector[] rows = new SparseRowVector[m2];
+ for(int j = 0; j < m2; j++)
+ rows[j] = new SparseRowVector(ennz2, n2);
+
+ if(k <= 1)
+ transposeDenseToSparseMMRange(a, rows, 0, m, 0, n);
+ else {
+ final ExecutorService pool = CommonThreadPool.get(k);
+ try {
+ final ArrayList<TransposeDenseToSparseTask>
tasks = new ArrayList<>();
+ final int rbz = Math.max(1, m2 / k);
+ for(int i = 0; i < m2; i += rbz)
+ tasks.add(new
TransposeDenseToSparseTask(a, rows, 0, m, i, Math.min(i + rbz, n)));
+ for(Future<Object> task : pool.invokeAll(tasks))
+ task.get();
+ pool.shutdown();
+ }
+ catch(Exception ex) {
+ pool.shutdown();
+ throw new DMLRuntimeException(ex);
+ }
}
- else //general case: MATRIX-MATRIX
- {
- //blocking according to typical L2 cache sizes
- final int blocksizeI = 128;
- final int blocksizeJ = 128;
-
- //blocked execution
- for( int bi = 0; bi<m; bi+=blocksizeI ) {
- int bimin = Math.min(bi+blocksizeI, m);
- for( int bj = 0; bj<n; bj+=blocksizeJ ) {
- int bjmin = Math.min(bj+blocksizeJ, n);
- //core transpose operation
- for( int i=bi; i<bimin; i++ ) {
- double[] avals = a.values(i);
- int aix = a.pos(i);
- for( int j=bj; j<bjmin; j++ ) {
- c.allocate(j, ennz2,
n2);
- c.append(j, i,
avals[aix+j]);
- }
- }
+
+ SparseBlock c = new SparseBlockMCSR(rows, false);
+ out.setSparseBlock(c);
+ }
+
+ private static void transposeDenseToSparseMMRange(DenseBlock a,
SparseRowVector[] rows, int rl, int ru, int cl,
+ int cu) {
+ // blocking according to typical L2 cache sizes
+ final int blocksizeI = 128;
+ final int blocksizeJ = 128;
+ for(int bi = rl; bi < ru; bi += blocksizeI) {
+ final int bimin = Math.min(bi + blocksizeI, ru);
+ for(int bj = cl; bj < cu; bj += blocksizeJ) {
+ final int bjmin = Math.min(bj + blocksizeJ, cu);
+ // core transpose operation
+ for(int i = bi; i < bimin; i++) {
+ final double[] avals = a.values(i);
+ final int aix = a.pos(i);
+ for(int j = bj; j < bjmin; j++)
+ rows[j].append(i, avals[aix +
j]);
}
}
}
}
+ private static class TransposeDenseToSparseTask implements
Callable<Object> {
+ private DenseBlock a;
+ private SparseRowVector[] rows;
+ private int rl;
+ private int ru;
+ private int cl;
+ private int cu;
+
+ protected TransposeDenseToSparseTask(DenseBlock a,
SparseRowVector[] rows, int rl, int ru, int cl, int cu) {
+ this.a = a;
+ this.rows = rows;
+ this.rl = rl;
+ this.ru = ru;
+ this.cl = cl;
+ this.cu = cu;
+ }
+
+ @Override
+ public Object call() {
+ transposeDenseToSparseMMRange(a, rows, rl, ru, cl, cu);
+ return null;
+ }
+ }
+
private static void transposeUltraSparse(MatrixBlock in, MatrixBlock
out) {
//note: applied if nnz < max(rlen, clen) - so no cache blocking
// but basic, naive transposition in a single-threaded context
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/readers/ReadersTest.java
b/src/test/java/org/apache/sysds/test/component/compress/readers/ReadersTest.java
index b7aeb9cd09..fa1db91818 100644
---
a/src/test/java/org/apache/sysds/test/component/compress/readers/ReadersTest.java
+++
b/src/test/java/org/apache/sysds/test/component/compress/readers/ReadersTest.java
@@ -41,7 +41,7 @@ public class ReadersTest {
@Test(expected = DMLCompressionException.class)
public void testDenseSingleCol() {
MatrixBlock mb = TestUtils.generateTestMatrixBlock(10, 1, 1, 1,
0.5, 21342);
- ReaderColumnSelection.createReader(mb,ColIndexFactory.create(
1), false);
+ ReaderColumnSelection.createReader(mb,
ColIndexFactory.create(1), false);
}
@Test
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/readers/ReadersTestCompareReaders.java
b/src/test/java/org/apache/sysds/test/component/compress/readers/ReadersTestCompareReaders.java
index 57f5d7866f..7f5d7990cf 100644
---
a/src/test/java/org/apache/sysds/test/component/compress/readers/ReadersTestCompareReaders.java
+++
b/src/test/java/org/apache/sysds/test/component/compress/readers/ReadersTestCompareReaders.java
@@ -117,7 +117,13 @@ public class ReadersTestCompareReaders {
sm = m2;
}
+ if(!sm.isInSparseFormat())
+ sm.denseToSparse(true);
+ if(m.isInSparseFormat())
+ m.sparseToDense();
+
MatrixBlock tin = LibMatrixReorg.transpose(in);
+
MatrixBlock m2t = new MatrixBlock();
m2t.copy(tin);
final boolean isSparseTIn = in.isInSparseFormat();
@@ -132,11 +138,22 @@ public class ReadersTestCompareReaders {
tsm = m2t;
}
+ if(tm.isInSparseFormat())
+ tm.sparseToDense();
+ if(!tsm.isInSparseFormat())
+ tsm.denseToSparse(true);
+
+ MatrixBlock tmd = new MatrixBlock();
+ tmd.copy(tm);
+ tmd.sparseToDense();
+
mMockLarge = new MatrixBlock(m.getNumRows(), m.getNumColumns(),
new DenseBlockFP64Mock(new int[] {m.getNumRows(),
m.getNumColumns()}, m.getDenseBlockValues()));
+ mMockLarge.setNonZeros(m.getNonZeros());
- mMockLargeTransposed = new MatrixBlock(tm.getNumRows(),
tm.getNumColumns(),
- new DenseBlockFP64Mock(new int[] {tm.getNumRows(),
tm.getNumColumns()}, tm.getDenseBlockValues()));
+ mMockLargeTransposed = new MatrixBlock(tmd.getNumRows(),
tmd.getNumColumns(),
+ new DenseBlockFP64Mock(new int[] {tmd.getNumRows(),
tmd.getNumColumns()}, tmd.getDenseBlockValues()));
+ mMockLargeTransposed.setNonZeros(tm.getNonZeros());
}
@@ -144,7 +161,8 @@ public class ReadersTestCompareReaders {
public void testCompareSparseDense() {
ReaderColumnSelection a = ReaderColumnSelection.createReader(m,
cols, false);
ReaderColumnSelection b =
ReaderColumnSelection.createReader(sm, cols, false);
- if(b instanceof ReaderColumnSelectionSparse && a instanceof
ReaderColumnSelectionDenseSingleBlock)
+ if((b instanceof ReaderColumnSelectionSparse && a instanceof
ReaderColumnSelectionDenseSingleBlock) ||
+ (m.isEmpty() || sm.isEmpty()))
compareReaders(a, b);
else
fail("Incorrect type of reader");
@@ -202,7 +220,7 @@ public class ReadersTestCompareReaders {
public void testCompareDenseTransposedDense() {
ReaderColumnSelection a = ReaderColumnSelection.createReader(m,
cols, false);
ReaderColumnSelection b =
ReaderColumnSelection.createReader(tm, cols, true);
- if(b instanceof ReaderColumnSelectionDenseSingleBlockTransposed)
+ if((b instanceof
ReaderColumnSelectionDenseSingleBlockTransposed) || (m.isEmpty() ||
tm.isEmpty()))
compareReaders(a, b);
else
fail("Incorrect type of reader");
@@ -248,7 +266,7 @@ public class ReadersTestCompareReaders {
public void testCompareDenseTransposedSparse() {
ReaderColumnSelection a = ReaderColumnSelection.createReader(m,
cols, false);
ReaderColumnSelection b =
ReaderColumnSelection.createReader(tsm, cols, true);
- if(b instanceof ReaderColumnSelectionSparseTransposed)
+ if((b instanceof ReaderColumnSelectionSparseTransposed) ||
(m.isEmpty() || tsm.isEmpty()))
compareReaders(a, b);
else
fail("Incorrect type of reader");
@@ -473,13 +491,20 @@ public class ReadersTestCompareReaders {
@Test
public void testCompareDenseTransposedLargeFewRowsFromEnd() {
- final int nRow = m.getNumRows();
- if(nRow > 30) {
- final int end = m.getNumRows() - 10;
- final int start = end - 10;
- ReaderColumnSelection a =
ReaderColumnSelection.createReader(m, cols, false, start, end);
- ReaderColumnSelection b =
ReaderColumnSelection.createReader(mMockLargeTransposed, cols, true, start,
end);
- compareReaders(a, b, start, end);
+ try {
+
+ final int nRow = m.getNumRows();
+ if(nRow > 30) {
+ final int end = m.getNumRows() - 10;
+ final int start = end - 10;
+ ReaderColumnSelection a =
ReaderColumnSelection.createReader(m, cols, false, start, end);
+ ReaderColumnSelection b =
ReaderColumnSelection.createReader(mMockLargeTransposed, cols, true, start,
end);
+ compareReaders(a, b, start, end);
+ }
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
}
}
diff --git
a/src/test/java/org/apache/sysds/test/component/matrix/TransposeCSRTest.java
b/src/test/java/org/apache/sysds/test/component/matrix/TransposeCSRTest.java
index 875667b726..9f4ff33719 100644
--- a/src/test/java/org/apache/sysds/test/component/matrix/TransposeCSRTest.java
+++ b/src/test/java/org/apache/sysds/test/component/matrix/TransposeCSRTest.java
@@ -79,7 +79,6 @@ public class TransposeCSRTest {
@Test
public void testEstimation() {
try {
-
LibMatrixReorg.PAR_NUMCELL_THRESHOLD = 100;
MatrixBlock ret1 = LibMatrixReorg.transpose(in,
new MatrixBlock(in.getNumColumns(),
in.getNumRows(), in.isInSparseFormat()), k, false);
diff --git
a/src/test/java/org/apache/sysds/test/component/matrix/TransposeTwiceTest.java
b/src/test/java/org/apache/sysds/test/component/matrix/TransposeTwiceTest.java
new file mode 100644
index 0000000000..6bae3f83ff
--- /dev/null
+++
b/src/test/java/org/apache/sysds/test/component/matrix/TransposeTwiceTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.matrix;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.runtime.matrix.data.LibMatrixReorg;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(value = Parameterized.class)
[email protected]
+public class TransposeTwiceTest {
+ protected static final Log LOG =
LogFactory.getLog(TransposeTwiceTest.class.getName());
+
+ long old = LibMatrixReorg.PAR_NUMCELL_THRESHOLD;
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ ArrayList<Object[]> tests = new ArrayList<>();
+
+ try {
+ double[] sparsities = new double[] {0.0, 0.001, 0.1,
0.2, 0.35, 0.5, 1.0};
+ int[] rowsCols = new int[] {1, 2, 4, 5, 10, 30, 50,
100, 300};
+ int[] parallelization = new int[] {-1, 1, 5};
+
+ // smaller cases
+ for(int rows : rowsCols) {
+ for(int cols : rowsCols) {
+ for(double sp : sparsities) {
+ MatrixBlock mb =
TestUtils.generateTestMatrixBlock(rows, cols, -10, 10, sp, 42);
+ mb.examSparsity(false);
+ for(int k1 : parallelization) {
+ for(int k2 :
parallelization) {
+ tests.add(new
Object[] {mb, k1, k2});
+ }
+ }
+ }
+ }
+ }
+
+ // Larger edge case
+ MatrixBlock mb;
+ rowsCols = new int[] {1, 2, 5, 10};
+ for(int rows : rowsCols) {
+ for(double sp : sparsities) {
+ if(sp > 0.4)
+ continue;
+ mb =
TestUtils.generateTestMatrixBlock(rows, 5000, -10, 10, sp, 43);
+ tests.add(new Object[] {mb, 16, 16});
+ }
+ }
+ }
+ catch(Exception e) {
+ fail(e.getMessage());
+ }
+ return tests;
+ }
+
+ @Parameterized.Parameter
+ public MatrixBlock in;
+ @Parameterized.Parameter(1)
+ public int k1;
+ @Parameterized.Parameter(2)
+ public int k2;
+
+ @Test
+ public void testTransposeBackAndForth() {
+ try {
+ LibMatrixReorg.PAR_NUMCELL_THRESHOLD = 100;
+
+ MatrixBlock inT = LibMatrixReorg.transpose(in, k1);
+ MatrixBlock inTT = LibMatrixReorg.transpose(inT, k2);
+
+ TestUtils.compareMatricesBitAvgDistance(in, inTT, 0, 0,
"Not equal transpose result");
+ assertEquals(in.getNonZeros(), inT.getNonZeros());
+ assertEquals(in.getNonZeros(), inTT.getNonZeros());
+ assertEquals(in.getNumRows(), inT.getNumColumns());
+ assertEquals(in.getNumColumns(), inT.getNumRows());
+ compareTransposed(in, inT);
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail("Failed in transposition");
+ }
+ finally {
+
+ LibMatrixReorg.PAR_NUMCELL_THRESHOLD = old;
+ }
+ }
+
+ private void compareTransposed(MatrixBlock in, MatrixBlock inT) {
+ int nRow = in.getNumRows();
+ int nCol = in.getNumColumns();
+ for(int i = 0; i < nRow; i++) {
+ for(int j = 0; j < nCol; j++) {
+ assertEquals(in.quickGetValue(i, j),
inT.quickGetValue(j, i), 0.0);
+ }
+ }
+ }
+}
