This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new b50885f4a8 [MINOR] Cleanup sparse block API (estimate size) and
warnings
b50885f4a8 is described below
commit b50885f4a8afc8c095ee4d87faddc2689117b7c2
Author: Matthias Boehm <[email protected]>
AuthorDate: Sat Jul 27 09:46:44 2024 +0200
[MINOR] Cleanup sparse block API (estimate size) and warnings
---
.../org/apache/sysds/runtime/data/SparseBlock.java | 6 +++
.../apache/sysds/runtime/data/SparseBlockCOO.java | 5 +--
.../apache/sysds/runtime/data/SparseBlockCSR.java | 5 +--
.../apache/sysds/runtime/data/SparseBlockDCSR.java | 5 +--
.../apache/sysds/runtime/data/SparseBlockMCSC.java | 6 +--
.../apache/sysds/runtime/data/SparseBlockMCSR.java | 5 +--
.../java/org/apache/sysds/utils/Statistics.java | 4 +-
.../org/apache/sysds/utils/stats/NGramBuilder.java | 1 +
.../sysds/performance/matrix/MatrixStorage.java | 50 ++++++----------------
.../test/applications/ApplyTransformTest.java | 5 ---
10 files changed, 29 insertions(+), 63 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java
b/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java
index ebe9e7bf04..648f42b690 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java
@@ -274,6 +274,12 @@ public abstract class SparseBlock implements Serializable,
Block
public abstract boolean checkValidity(int rlen, int clen, long nnz,
boolean strict);
+ /**
+ * Computes the exact size in memory of the materialized block
+ * @return the exact size in memory
+ */
+ public abstract long getExactSizeInMemory();
+
////////////////////////
//obtain indexes/values/positions
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java
b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java
index d83a9263dc..c4e60c10cf 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java
@@ -158,10 +158,7 @@ public class SparseBlockCOO extends SparseBlock
return (long) Math.min(size, Long.MAX_VALUE);
}
- /**
- * Computes the exact size in memory of the materialized block
- * @return the exact size in memory
- */
+ @Override
public long getExactSizeInMemory() {
//32B overhead per array, int/int/double arr in nnz
double size = 16 + 8; //object + 2 int fields
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java
b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java
index 68cc6c8f86..cfa1258d37 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java
@@ -285,10 +285,7 @@ public class SparseBlockCSR extends SparseBlock
return (long) Math.min(size, Long.MAX_VALUE);
}
- /**
- * Computes the exact size in memory of the materialized block
- * @return the exact size in memory
- */
+ @Override
public long getExactSizeInMemory() {
//32B overhead per array, int arr in nrows, int/double arr in
nnz
double size = 16 + 4 + 4;
//object + int field + padding
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockDCSR.java
b/src/main/java/org/apache/sysds/runtime/data/SparseBlockDCSR.java
index 3029370d63..b369992efa 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockDCSR.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockDCSR.java
@@ -175,10 +175,7 @@ public class SparseBlockDCSR extends SparseBlock
return (long) Math.min(size, Long.MAX_VALUE);
}
- /**
- * Computes the exact size in memory of the materialized block
- * @return the exact size in memory
- */
+ @Override
public long getExactSizeInMemory() {
double size = 16;
size += 4 + 4 + 4 + 4;
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSC.java
b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSC.java
index 7a12d3e7aa..fd0b3906bc 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSC.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSC.java
@@ -211,11 +211,7 @@ public class SparseBlockMCSC extends SparseBlock {
return (long) Math.min(size, Long.MAX_VALUE);
}
- /**
- * Computes the exact size in memory of the materialized block
- *
- * @return the exact size in memory
- */
+ @Override
public long getExactSizeInMemory() {
double size = 16; //object
size += MemoryEstimates.objectArrayCost(_columns.length);
//references
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
index d232a0041a..f94b6bf7f4 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
@@ -136,10 +136,7 @@ public class SparseBlockMCSR extends SparseBlock
return (long) Math.min(size, Long.MAX_VALUE);
}
- /**
- * Computes the exact size in memory of the materialized block
- * @return the exact size in memory
- */
+ @Override
public long getExactSizeInMemory() {
double size = 16; //object
size += MemoryEstimates.objectArrayCost(_rows.length);
//references
diff --git a/src/main/java/org/apache/sysds/utils/Statistics.java
b/src/main/java/org/apache/sysds/utils/Statistics.java
index a7c764cf78..3ad613c842 100644
--- a/src/main/java/org/apache/sysds/utils/Statistics.java
+++ b/src/main/java/org/apache/sysds/utils/Statistics.java
@@ -401,6 +401,7 @@ public class Statistics
tmp.count.increment();
}
+ @SuppressWarnings("unchecked")
public static void maintainNGrams(String instName, long timeNanos) {
NGramBuilder<String, NGramStats>[] tmp =
_instStatsNGram.computeIfAbsent(Thread.currentThread().getName(), k -> {
NGramBuilder<String, NGramStats>[] threadEntry = new
NGramBuilder[DMLScript.STATISTICS_NGRAM_SIZES.length];
@@ -414,6 +415,7 @@ public class Statistics
tmp[i].append(instName, new NGramStats(1, timeNanos,
0));
}
+ @SuppressWarnings("unchecked")
public static NGramBuilder<String, NGramStats>[] mergeNGrams() {
NGramBuilder<String, NGramStats>[] builders = new
NGramBuilder[DMLScript.STATISTICS_NGRAM_SIZES.length];
@@ -545,7 +547,7 @@ public class Statistics
//long timeNs = tmp[len - 1 -
i].getValue().time.longValue();
double timeS = topNGrams[i].getCumStats().cumTimeNanos
/ 1000000000d;
- double timeVar =
topNGrams[i].getCumStats().getTimeVariance();
+ //double timeVar =
topNGrams[i].getCumStats().getTimeVariance();
String timeSString = sFormat.format(timeS);
String timeVarString =
getNGramStdDevs(topNGrams[i].getStats(), topNGrams[i].getOffset(), 3,
false);//sFormat.format(timeVar);
diff --git a/src/main/java/org/apache/sysds/utils/stats/NGramBuilder.java
b/src/main/java/org/apache/sysds/utils/stats/NGramBuilder.java
index 7554fdcd67..e0212e5c73 100644
--- a/src/main/java/org/apache/sysds/utils/stats/NGramBuilder.java
+++ b/src/main/java/org/apache/sysds/utils/stats/NGramBuilder.java
@@ -115,6 +115,7 @@ public class NGramBuilder<T, U> {
return this;
}
+ @SuppressWarnings("unused")
private NGramEntry<T, U> add(NGramEntry<T, U> entry) {
return add(entry.occurrences);
}
diff --git
a/src/test/java/org/apache/sysds/performance/matrix/MatrixStorage.java
b/src/test/java/org/apache/sysds/performance/matrix/MatrixStorage.java
index 1a911fed7b..c23b157799 100644
--- a/src/test/java/org/apache/sysds/performance/matrix/MatrixStorage.java
+++ b/src/test/java/org/apache/sysds/performance/matrix/MatrixStorage.java
@@ -21,10 +21,7 @@ package org.apache.sysds.performance.matrix;
import org.apache.sysds.runtime.data.DenseBlockFP64;
import org.apache.sysds.runtime.data.SparseBlock;
-import org.apache.sysds.runtime.data.SparseBlockCOO;
-import org.apache.sysds.runtime.data.SparseBlockCSR;
-import org.apache.sysds.runtime.data.SparseBlockDCSR;
-import org.apache.sysds.runtime.data.SparseBlockMCSR;
+import org.apache.sysds.runtime.data.SparseBlockFactory;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.util.DataConverter;
import org.apache.sysds.test.TestUtils;
@@ -183,37 +180,18 @@ public class MatrixStorage {
}
private long evaluateMemoryConsumption(SparseBlock.Type btype, double
sparsity, int rl, int cl) {
- try
- {
- if (btype == null)
- return Math.min(Long.MAX_VALUE, (long)
DenseBlockFP64.estimateMemory(rl, cl));
-
- double[][] A = TestUtils.generateTestMatrix(rl, cl,
-10, 10, sparsity, 7654321);
-
- MatrixBlock mbtmp =
DataConverter.convertToMatrixBlock(A);
-
- if (!mbtmp.isInSparseFormat())
- mbtmp.denseToSparse(true);
-
- SparseBlock srtmp = mbtmp.getSparseBlock();
- switch (btype) {
- case MCSR:
- SparseBlockMCSR mcsr = new
SparseBlockMCSR(srtmp);
- return mcsr.getExactSizeInMemory();
- case CSR:
- SparseBlockCSR csr = new
SparseBlockCSR(srtmp);
- return csr.getExactSizeInMemory();
- case COO:
- SparseBlockCOO coo = new
SparseBlockCOO(srtmp);
- return coo.getExactSizeInMemory();
- case DCSR:
- SparseBlockDCSR dcsr = new
SparseBlockDCSR(srtmp);
- return dcsr.getExactSizeInMemory();
- }
- } catch(Exception ex) {
- ex.printStackTrace();
- throw new RuntimeException(ex);
- }
- throw new IllegalArgumentException();
+ if (btype == null)
+ return Math.min(Long.MAX_VALUE, (long)
DenseBlockFP64.estimateMemory(rl, cl));
+
+ double[][] A = TestUtils.generateTestMatrix(rl, cl, -10, 10,
sparsity, 7654321);
+
+ MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A);
+
+ if (!mbtmp.isInSparseFormat())
+ mbtmp.denseToSparse(true);
+
+ SparseBlock srtmp = mbtmp.getSparseBlock();
+ SparseBlock sb = SparseBlockFactory.copySparseBlock(btype,
srtmp, true);
+ return sb.getExactSizeInMemory();
}
}
diff --git
a/src/test/java/org/apache/sysds/test/applications/ApplyTransformTest.java
b/src/test/java/org/apache/sysds/test/applications/ApplyTransformTest.java
index bdd0a9b415..9914caa6ec 100644
--- a/src/test/java/org/apache/sysds/test/applications/ApplyTransformTest.java
+++ b/src/test/java/org/apache/sysds/test/applications/ApplyTransformTest.java
@@ -19,8 +19,6 @@
package org.apache.sysds.test.applications;
-import java.io.FileWriter;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -29,9 +27,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.sysds.api.DMLScript;
-import org.apache.sysds.utils.Statistics;
-import org.apache.sysds.utils.stats.NGramBuilder;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
