This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 1f17b56d8c [MINOR] Add DataTypes on memory estimates for HOPS
1f17b56d8c is described below
commit 1f17b56d8ca053d39e1046309e3d9b6827e43787
Author: Sebastian Baunsgaard <[email protected]>
AuthorDate: Wed Feb 5 20:50:44 2025 +0100
[MINOR] Add DataTypes on memory estimates for HOPS
This commit changes the memory estimates of many of the HOPS,
to include the datatype, Indicating if the output is a frame or a
Matrix. It is included because I had to make one modification to a
unary Op in BWARE, and in general, it does not hurt to add to many of the
Op Types (even if they do not return frame types at the moment)
Closes #2221
---
src/main/java/org/apache/sysds/hops/BinaryOp.java | 2 +-
src/main/java/org/apache/sysds/hops/DataGenOp.java | 4 ++--
src/main/java/org/apache/sysds/hops/DataOp.java | 2 +-
src/main/java/org/apache/sysds/hops/IndexingOp.java | 2 +-
src/main/java/org/apache/sysds/hops/OptimizerUtils.java | 10 +++++++++-
src/main/java/org/apache/sysds/hops/ReorgOp.java | 2 +-
src/main/java/org/apache/sysds/hops/UnaryOp.java | 2 +-
7 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/src/main/java/org/apache/sysds/hops/BinaryOp.java
b/src/main/java/org/apache/sysds/hops/BinaryOp.java
index cb20df8366..a3ddb45ea6 100644
--- a/src/main/java/org/apache/sysds/hops/BinaryOp.java
+++ b/src/main/java/org/apache/sysds/hops/BinaryOp.java
@@ -578,7 +578,7 @@ public class BinaryOp extends MultiThreadedHop {
else //e.g., for append,pow or after inference
sparsity = OptimizerUtils.getSparsity(dim1,
dim2, nnz);
- ret = OptimizerUtils.estimateSizeExactSparsity(dim1,
dim2, sparsity);
+ ret = OptimizerUtils.estimateSizeExactSparsity(dim1,
dim2, sparsity, getDataType());
}
return ret;
}
diff --git a/src/main/java/org/apache/sysds/hops/DataGenOp.java
b/src/main/java/org/apache/sysds/hops/DataGenOp.java
index fd3ecb97fa..b01d6ed381 100644
--- a/src/main/java/org/apache/sysds/hops/DataGenOp.java
+++ b/src/main/java/org/apache/sysds/hops/DataGenOp.java
@@ -219,11 +219,11 @@ public class DataGenOp extends MultiThreadedHop
else {
//sparsity-aware estimation (dependent on
sparse generation approach); for pure dense generation
//we would need to disable sparsity-awareness
and estimate via sparsity=1.0
- ret =
OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, _sparsity);
+ ret =
OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, _sparsity, getDataType());
}
}
else {
- ret = OptimizerUtils.estimateSizeExactSparsity(dim1,
dim2, 1.0);
+ ret = OptimizerUtils.estimateSizeExactSparsity(dim1,
dim2, 1.0, getDataType());
}
return ret;
diff --git a/src/main/java/org/apache/sysds/hops/DataOp.java
b/src/main/java/org/apache/sysds/hops/DataOp.java
index 7be61f4129..1ae8616001 100644
--- a/src/main/java/org/apache/sysds/hops/DataOp.java
+++ b/src/main/java/org/apache/sysds/hops/DataOp.java
@@ -390,7 +390,7 @@ public class DataOp extends Hop {
|| _op == OpOpData.TRANSIENTREAD )
{
double sparsity =
OptimizerUtils.getSparsity(dim1, dim2, nnz);
- ret =
OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
+ ret =
OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity, getDataType());
}
// output memory estimate is not required for "write"
nodes (just input)
}
diff --git a/src/main/java/org/apache/sysds/hops/IndexingOp.java
b/src/main/java/org/apache/sysds/hops/IndexingOp.java
index 457c5b44e0..22e0b30e0d 100644
--- a/src/main/java/org/apache/sysds/hops/IndexingOp.java
+++ b/src/main/java/org/apache/sysds/hops/IndexingOp.java
@@ -204,7 +204,7 @@ public class IndexingOp extends Hop
{
// only dense right indexing supported on GPU
double sparsity = isGPUEnabled() ? 1.0 :
OptimizerUtils.getSparsity(dim1, dim2, nnz);
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2,
sparsity);
+ return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2,
sparsity, getDataType());
}
@Override
diff --git a/src/main/java/org/apache/sysds/hops/OptimizerUtils.java
b/src/main/java/org/apache/sysds/hops/OptimizerUtils.java
index 30df23dba2..73dd4a80a3 100644
--- a/src/main/java/org/apache/sysds/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysds/hops/OptimizerUtils.java
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.sysds.api.DMLScript;
+import org.apache.sysds.common.Types.DataType;
import org.apache.sysds.common.Types.ExecMode;
import org.apache.sysds.common.Types.ExecType;
import org.apache.sysds.common.Types.FileFormat;
@@ -63,8 +64,8 @@ import org.apache.sysds.runtime.meta.DataCharacteristics;
import org.apache.sysds.runtime.meta.MatrixCharacteristics;
import org.apache.sysds.runtime.util.IndexRange;
import org.apache.sysds.runtime.util.UtilFunctions;
-import org.apache.sysds.utils.stats.InfrastructureAnalyzer;
import org.apache.sysds.utils.MemoryEstimates;
+import org.apache.sysds.utils.stats.InfrastructureAnalyzer;
public class OptimizerUtils
{
@@ -822,6 +823,13 @@ public class OptimizerUtils
return MatrixBlock.estimateSizeInMemory(nrows,ncols,sp);
}
+ public static long estimateSizeExactSparsity(long nrows, long ncols,
double sp, DataType dt){
+ if(dt == DataType.FRAME)
+ return estimateSizeExactFrame(nrows, ncols);
+ else
+ return estimateSizeExactSparsity(nrows, ncols, sp);
+ }
+
/**
* Estimates the footprint (in bytes) for a partitioned in-memory
representation of a
* matrix with the given matrix characteristics
diff --git a/src/main/java/org/apache/sysds/hops/ReorgOp.java
b/src/main/java/org/apache/sysds/hops/ReorgOp.java
index 576ccaa83a..df6b4381ae 100644
--- a/src/main/java/org/apache/sysds/hops/ReorgOp.java
+++ b/src/main/java/org/apache/sysds/hops/ReorgOp.java
@@ -232,7 +232,7 @@ public class ReorgOp extends MultiThreadedHop
protected double computeOutputMemEstimate( long dim1, long dim2, long
nnz ) {
//no dedicated mem estimation per op type, because always
propagated via refreshSizeInformation
double sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2,
sparsity);
+ return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2,
sparsity, getDataType());
}
@Override
diff --git a/src/main/java/org/apache/sysds/hops/UnaryOp.java
b/src/main/java/org/apache/sysds/hops/UnaryOp.java
index 91f3a5ec58..34da36dd13 100644
--- a/src/main/java/org/apache/sysds/hops/UnaryOp.java
+++ b/src/main/java/org/apache/sysds/hops/UnaryOp.java
@@ -366,7 +366,7 @@ public class UnaryOp extends MultiThreadedHop
} else {
sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
}
- return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2,
sparsity);
+ return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2,
sparsity, getDataType());
}
@Override
