This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new a94b4c8 [SYSTEMDS-3297] Fix reshape of misaligned dense large blocks
a94b4c8 is described below
commit a94b4c8a7527f047d5722744b2cc95bb620c0025
Author: Matthias Boehm <[email protected]>
AuthorDate: Fri Feb 25 23:14:08 2022 +0100
[SYSTEMDS-3297] Fix reshape of misaligned dense large blocks
The row-wise reshape of dense large blocks (>16GB) assumed aligned
physical blocksizes. This is however only the case in special cases and
otherwise the reshape needs to handle rows on input or output that
stretch multiple physical blocks on the other side. At the same time it
need to work with different input/output value types.
Besides the extended implementation, this fix also adds a performance
improvement by additional specialization for DenseBlockLFP64.
---
.../apache/sysds/runtime/data/DenseBlockLDRB.java | 33 ++++++++++++++++++----
.../apache/sysds/runtime/data/DenseBlockLFP64.java | 20 +++++++++++++
2 files changed, 48 insertions(+), 5 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/data/DenseBlockLDRB.java
b/src/main/java/org/apache/sysds/runtime/data/DenseBlockLDRB.java
index 2da57e0..339dbb5 100644
--- a/src/main/java/org/apache/sysds/runtime/data/DenseBlockLDRB.java
+++ b/src/main/java/org/apache/sysds/runtime/data/DenseBlockLDRB.java
@@ -200,11 +200,34 @@ public abstract class DenseBlockLDRB extends DenseBlock
public DenseBlock set(DenseBlock db) {
// ToDo: Optimize if dense block types match
// ToDo: Performance
- for (int ri = 0; ri < _rlen; ri += blockSize()) {
- int bix = ri / blockSize();
- double[] other = db.valuesAt(bix);
- IntStream.range(0, blockSize(bix) * _odims[0])
- .forEach((i) -> setInternal(bix, i,
other[i]));
+
+ // this implementation needs to be robust against rows in the
input
+ // stretching over multiple blocks in the output and vice versa
+ boolean aligned = blockSize()*_odims[0] ==
db.blockSize()*db._odims[0];
+ if( aligned ) {
+ for (int ri = 0; ri < _rlen; ri += blockSize()) {
+ int bix = ri / blockSize();
+ double[] other = db.valuesAt(bix);
+ IntStream.range(0, blockSize(bix) * _odims[0])
+ .forEach(i -> setInternal(bix, i,
other[i]));
+ }
+ }
+ else {
+ long globalPos = 0;
+ int bsize = blockSize() * _odims[0];
+ for( int bix = 0; bix < db.numBlocks(); bix++ ) {
+ double[] other = db.valuesAt(bix);
+ int blen = db.blockSize(bix) * db._odims[0];
+ int bix2 = (int)(globalPos / bsize);
+ int off2 = (int)(globalPos % bsize);
+ int blen2 = size(bix2);
+ for( int i = 0; i < Math.min(blen,blen2-off2);
i++ )
+ setInternal(bix2, off2+i, other[i]);
+ if( blen2-off2 < blen )
+ for( int i = blen2-off2; i < blen; i++ )
+ setInternal(bix2+1,
i-(blen2-off2), other[i]);
+ globalPos += blen;
+ }
}
return this;
}
diff --git a/src/main/java/org/apache/sysds/runtime/data/DenseBlockLFP64.java
b/src/main/java/org/apache/sysds/runtime/data/DenseBlockLFP64.java
index 8daf41a..4634a96 100644
--- a/src/main/java/org/apache/sysds/runtime/data/DenseBlockLFP64.java
+++ b/src/main/java/org/apache/sysds/runtime/data/DenseBlockLFP64.java
@@ -142,4 +142,24 @@ public class DenseBlockLFP64 extends DenseBlockLDRB
public long getLong(int[] ix) {
return UtilFunctions.toLong(_blocks[index(ix[0])][pos(ix)]);
}
+
+ @Override
+ public DenseBlock set(DenseBlock db) {
+ // this implementation needs to be robust against rows in the
input
+ // stretching over multiple blocks in the output and vice versa
+ long globalPos = 0;
+ int bsize = blockSize() * _odims[0];
+ for( int bix = 0; bix < db.numBlocks(); bix++ ) {
+ double[] other = db.valuesAt(bix);
+ int blen = db.blockSize(bix) * db._odims[0];
+ int bix2 = (int)(globalPos / bsize);
+ int off2 = (int)(globalPos % bsize);
+ int blen2 = size(bix2);
+ System.arraycopy(other, 0, valuesAt(bix2), off2,
Math.min(blen,blen2-off2));
+ if( blen2-off2 < blen )
+ System.arraycopy(other, blen2-off2,
valuesAt(bix2+1), 0, blen-(blen2-off2));
+ globalPos += blen;
+ }
+ return this;
+ }
}
