This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new cebf845cac [MINOR] Misc code cleanups for improve test code coverage
cebf845cac is described below
commit cebf845cac2753f8b9cf459ee023c6ddbc0d35bb
Author: Matthias Boehm <[email protected]>
AuthorDate: Fri Nov 29 18:40:28 2024 +0100
[MINOR] Misc code cleanups for improve test code coverage
---
.../java/org/apache/sysds/hops/AggBinaryOp.java | 23 +-
.../java/org/apache/sysds/hops/AggUnaryOp.java | 36 +--
src/main/java/org/apache/sysds/hops/BinaryOp.java | 22 +-
src/main/java/org/apache/sysds/hops/DataGenOp.java | 7 -
src/main/java/org/apache/sysds/hops/DataOp.java | 27 --
src/main/java/org/apache/sysds/hops/DnnOp.java | 5 -
.../java/org/apache/sysds/hops/FunctionOp.java | 9 +-
src/main/java/org/apache/sysds/hops/Hop.java | 26 +-
.../java/org/apache/sysds/hops/IndexingOp.java | 5 -
.../java/org/apache/sysds/hops/LeftIndexingOp.java | 5 -
src/main/java/org/apache/sysds/hops/LiteralOp.java | 5 -
src/main/java/org/apache/sysds/hops/NaryOp.java | 4 -
.../apache/sysds/hops/ParameterizedBuiltinOp.java | 7 -
.../java/org/apache/sysds/hops/QuaternaryOp.java | 6 -
src/main/java/org/apache/sysds/hops/ReorgOp.java | 21 --
src/main/java/org/apache/sysds/hops/TernaryOp.java | 11 -
src/main/java/org/apache/sysds/hops/UnaryOp.java | 5 -
.../apache/sysds/hops/codegen/SpoofFusedOp.java | 3 -
.../apache/sysds/hops/ipa/FunctionCallGraph.java | 6 +-
.../apache/sysds/hops/rewrite/HopDagValidator.java | 3 -
.../spark/functions/GetMIMBFromRow.java | 40 ---
.../sysds/runtime/matrix/data/CM_N_COVCell.java | 273 ---------------------
.../sysds/runtime/matrix/data/MatrixCell.java | 116 ++-------
.../matrix/data/TextToBinaryCellConverter.java | 86 -------
.../sysds/runtime/matrix/data/WeightedPair.java | 113 ---------
.../runtime/matrix/operators/ReIndexOperator.java | 30 ---
.../sysds/runtime/meta/DataCharacteristics.java | 16 --
.../test/component/utils/HopDagValidationTest.java | 91 +++++++
28 files changed, 142 insertions(+), 859 deletions(-)
diff --git a/src/main/java/org/apache/sysds/hops/AggBinaryOp.java
b/src/main/java/org/apache/sysds/hops/AggBinaryOp.java
index 640ababbad..2cf651f189 100644
--- a/src/main/java/org/apache/sysds/hops/AggBinaryOp.java
+++ b/src/main/java/org/apache/sysds/hops/AggBinaryOp.java
@@ -111,11 +111,6 @@ public class AggBinaryOp extends MultiThreadedHop {
refreshSizeInformation();
}
- @Override
- public void checkArity() {
- HopsException.check(_input.size() == 2, this, "should have
arity 2 but has arity %d", _input.size());
- }
-
public void setHasLeftPMInput(boolean flag) {
_hasLeftPMInput = flag;
}
@@ -387,26 +382,24 @@ public class AggBinaryOp extends MultiThreadedHop {
{
checkAndSetForcedPlatform();
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
+ if( _etypeForced != null ) {
+ setExecType(_etypeForced);
}
else
{
- if ( OptimizerUtils.isMemoryBasedOptLevel() )
- {
- _etype = findExecTypeByMemEstimate();
+ if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
+ setExecType(findExecTypeByMemEstimate());
}
// choose CP if the dimensions of both inputs are below
Hops.CPThreshold
// OR if it is vector-vector inner product
else if ( (getInput().get(0).areDimsBelowThreshold() &&
getInput().get(1).areDimsBelowThreshold())
||
(getInput().get(0).isVector() && getInput().get(1).isVector() &&
!isOuterProduct()) )
{
- _etype = ExecType.CP;
+ setExecType(ExecType.CP);
}
else
{
- _etype = ExecType.SPARK;
+ setExecType(ExecType.SPARK);
}
//check for valid CP mmchain, send invalid memory
requirements to remote
@@ -414,7 +407,7 @@ public class AggBinaryOp extends MultiThreadedHop {
&& checkMapMultChain() != ChainType.NONE
&& OptimizerUtils.getLocalMemBudget() <
getInput().get(0).getInput().get(0).getOutputMemEstimate() ) {
- _etype = ExecType.SPARK;
+ setExecType(ExecType.SPARK);
}
//check for valid CP dimensions and matrix size
@@ -429,7 +422,7 @@ public class AggBinaryOp extends MultiThreadedHop {
|| ( !mmtsj.isRight() &&
isApplicableForTransitiveSparkExecType(false))) )
{
//pull binary aggregate into spark
- _etype = ExecType.SPARK;
+ setExecType(ExecType.SPARK);
}
//mark for recompile (forever)
diff --git a/src/main/java/org/apache/sysds/hops/AggUnaryOp.java
b/src/main/java/org/apache/sysds/hops/AggUnaryOp.java
index 954caa7a2e..954114a0a4 100644
--- a/src/main/java/org/apache/sysds/hops/AggUnaryOp.java
+++ b/src/main/java/org/apache/sysds/hops/AggUnaryOp.java
@@ -63,28 +63,19 @@ public class AggUnaryOp extends MultiThreadedHop
inp.getParent().add(this);
}
- @Override
- public void checkArity() {
- HopsException.check(_input.size() == 1, this, "should have
arity 1 but has arity %d", _input.size());
- }
-
- public AggOp getOp()
- {
+ public AggOp getOp() {
return _op;
}
- public void setOp(AggOp op)
- {
+ public void setOp(AggOp op) {
_op = op;
}
- public Direction getDirection()
- {
+ public Direction getDirection() {
return _direction;
}
- public void setDirection(Direction direction)
- {
+ public void setDirection(Direction direction) {
_direction = direction;
}
@@ -370,9 +361,8 @@ public class AggUnaryOp extends MultiThreadedHop
ExecType REMOTE = ExecType.SPARK;
//forced / memory-based / threshold-based decision
- if( _etypeForced != null )
- {
- _etype = _etypeForced;
+ if( _etypeForced != null ) {
+ setExecType(_etypeForced);
}
else
{
@@ -381,10 +371,10 @@ public class AggUnaryOp extends MultiThreadedHop
}
// Choose CP, if the input dimensions are below
threshold or if the input is a vector
else if(getInput().get(0).areDimsBelowThreshold() ||
getInput().get(0).isVector()) {
- _etype = ExecType.CP;
+ setExecType(ExecType.CP);
}
else {
- _etype = REMOTE;
+ setExecType(REMOTE);
}
//check for valid CP dimensions and matrix size
@@ -402,15 +392,15 @@ public class AggUnaryOp extends MultiThreadedHop
&& (onlyOneParent() || allParentsSpark() ||
inputDoesNotRequireAggregation() ))
{
//pull unary aggregate into spark
- _etype = ExecType.SPARK;
+ setExecType(ExecType.SPARK);
}
//ensure cp exec type for single-node operations
- if( _op == AggOp.UNIQUE ) {
- _etype = ExecType.CP;
- } else {
+ if( _op == AggOp.UNIQUE )
+ setExecType(ExecType.CP);
+ else
setRequiresRecompileIfNecessary();
- }
+
return _etype;
}
diff --git a/src/main/java/org/apache/sysds/hops/BinaryOp.java
b/src/main/java/org/apache/sysds/hops/BinaryOp.java
index a47d6238be..839ce641af 100644
--- a/src/main/java/org/apache/sysds/hops/BinaryOp.java
+++ b/src/main/java/org/apache/sysds/hops/BinaryOp.java
@@ -110,11 +110,6 @@ public class BinaryOp extends MultiThreadedHop {
refreshSizeInformation();
}
- @Override
- public void checkArity() {
- HopsException.check(_input.size() == 2, this, "should have
arity 2 but has arity %d", _input.size());
- }
-
public OpOp2 getOp() {
return op;
}
@@ -756,13 +751,12 @@ public class BinaryOp extends MultiThreadedHop {
DataType dt2 = getInput().get(1).getDataType();
if( _etypeForced != null ) {
- _etype = _etypeForced;
+ setExecType(_etypeForced);
}
else
{
- if ( OptimizerUtils.isMemoryBasedOptLevel() )
- {
- _etype = findExecTypeByMemEstimate();
+ if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
+ setExecType(findExecTypeByMemEstimate());
}
else
{
@@ -773,29 +767,29 @@ public class BinaryOp extends MultiThreadedHop {
if (
(getInput().get(0).areDimsBelowThreshold() &&
getInput().get(1).areDimsBelowThreshold())
||
(getInput().get(0).isVector() && getInput().get(1).isVector()))
{
- _etype = ExecType.CP;
+ setExecType(ExecType.CP);
}
}
else if ( dt1 == DataType.MATRIX && dt2 ==
DataType.SCALAR ) {
if (
getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector() )
{
- _etype = ExecType.CP;
+ setExecType(ExecType.CP);
}
}
else if ( dt1 == DataType.SCALAR && dt2 ==
DataType.MATRIX ) {
if (
getInput().get(1).areDimsBelowThreshold() || getInput().get(1).isVector() )
{
- _etype = ExecType.CP;
+ setExecType(ExecType.CP);
}
}
else
{
- _etype = ExecType.CP;
+ setExecType(ExecType.CP);
}
//if no CP condition applied
if( _etype == null )
- _etype = ExecType.SPARK;
+ setExecType(ExecType.SPARK);
}
//check for valid CP dimensions and matrix size
diff --git a/src/main/java/org/apache/sysds/hops/DataGenOp.java
b/src/main/java/org/apache/sysds/hops/DataGenOp.java
index 8bab19be1b..fd3ecb97fa 100644
--- a/src/main/java/org/apache/sysds/hops/DataGenOp.java
+++ b/src/main/java/org/apache/sysds/hops/DataGenOp.java
@@ -140,13 +140,6 @@ public class DataGenOp extends MultiThreadedHop
refreshSizeInformation();
}
- @Override
- public void checkArity() {
- int sz = _input.size();
- int pz = _paramIndexMap.size();
- HopsException.check(sz == pz, this, "has %d inputs but %d
parameters", sz, pz);
- }
-
@Override
public String getOpString() {
return "dg(" + _op.toString().toLowerCase() +")";
diff --git a/src/main/java/org/apache/sysds/hops/DataOp.java
b/src/main/java/org/apache/sysds/hops/DataOp.java
index 62d20fce3c..82e5ecbbad 100644
--- a/src/main/java/org/apache/sysds/hops/DataOp.java
+++ b/src/main/java/org/apache/sysds/hops/DataOp.java
@@ -200,33 +200,6 @@ public class DataOp extends Hop {
setFileFormat(FileFormat.BINARY);
}
- /** Check for N (READ) or N+1 (WRITE) inputs. */
- @Override
- public void checkArity() {
- int sz = _input.size();
- int pz = _paramIndexMap.size();
- switch (_op) {
- case PERSISTENTREAD:
- case TRANSIENTREAD:
- case SQLREAD:
- HopsException.check(sz == pz, this,
- "in %s operator type has %d inputs and
%d parameters", _op.name(), sz, pz);
- break;
- case PERSISTENTWRITE:
- case TRANSIENTWRITE:
- case FUNCTIONOUTPUT:
- HopsException.check(sz == pz + 1, this,
- "in %s operator type has %d
inputs and %d parameters (expect 1 more input for write operator type)",
- _op.name(), sz, pz);
- break;
-
- case FEDERATED:
- //TODO
- default:
- //do nothing
- }
- }
-
public OpOpData getOp() {
return _op;
}
diff --git a/src/main/java/org/apache/sysds/hops/DnnOp.java
b/src/main/java/org/apache/sysds/hops/DnnOp.java
index 1600b5298a..90074321cd 100644
--- a/src/main/java/org/apache/sysds/hops/DnnOp.java
+++ b/src/main/java/org/apache/sysds/hops/DnnOp.java
@@ -84,11 +84,6 @@ public class DnnOp extends MultiThreadedHop {
refreshSizeInformation();
}
- @Override
- public void checkArity() {
- HopsException.check(_input.size() >= 1, this, "should have at
least one input but has %d inputs", _input.size());
- }
-
public OpOpDnn getOp() {
return op;
}
diff --git a/src/main/java/org/apache/sysds/hops/FunctionOp.java
b/src/main/java/org/apache/sysds/hops/FunctionOp.java
index f612440075..231438c7d4 100644
--- a/src/main/java/org/apache/sysds/hops/FunctionOp.java
+++ b/src/main/java/org/apache/sysds/hops/FunctionOp.java
@@ -91,10 +91,6 @@ public class FunctionOp extends MultiThreadedHop
}
}
- /** FunctionOps may have any number of inputs. */
- @Override
- public void checkArity() {}
-
public String getFunctionKey() {
return DMLProgram.constructFunctionKey(
getFunctionNamespace(), getFunctionName());
@@ -403,10 +399,11 @@ public class FunctionOp extends MultiThreadedHop
if ( getFunctionType() == FunctionType.MULTIRETURN_BUILTIN ) {
boolean isBuiltinFunction = isBuiltinFunction();
// check if there is sufficient memory to execute this
function
+ double mem = getMemEstimate(); // obtain memory
estimates for all (e.g., used in parfor)
if(isBuiltinFunction &&
getFunctionName().equalsIgnoreCase("transformencode") ) {
_etype = ((_etypeForced==ExecType.SPARK
- || (getMemEstimate() >=
OptimizerUtils.getLocalMemBudget()
- &&
OptimizerUtils.isSparkExecutionMode())) ? ExecType.SPARK : ExecType.CP);
+ || (mem >=
OptimizerUtils.getLocalMemBudget() && OptimizerUtils.isSparkExecutionMode())) ?
+ ExecType.SPARK : ExecType.CP);
}
else if(isBuiltinFunction &&
(getFunctionName().equalsIgnoreCase("lstm") ||
getFunctionName().equalsIgnoreCase("lstm_backward"))) {
_etype = DMLScript.USE_ACCELERATOR ?
ExecType.GPU : ExecType.CP;
diff --git a/src/main/java/org/apache/sysds/hops/Hop.java
b/src/main/java/org/apache/sysds/hops/Hop.java
index 44930604b3..b32a1a74aa 100644
--- a/src/main/java/org/apache/sysds/hops/Hop.java
+++ b/src/main/java/org/apache/sysds/hops/Hop.java
@@ -181,19 +181,7 @@ public abstract class Hop implements ParseInfo {
return _ID;
}
- /**
- * Check whether this Hop has a correct number of inputs.
- *
- * (Some Hops can have a variable number of inputs, such as DataOp,
DataGenOp, ParameterizedBuiltinOp,
- * ReorgOp, TernaryOp, QuaternaryOp, MultipleOp, DnnOp, and
SpoofFusedOp.)
- *
- * Parameterized Hops (such as DataOp) can check that the number of
parameters matches the number of inputs.
- *
- */
- public abstract void checkArity();
-
- public ExecType getExecType()
- {
+ public ExecType getExecType() {
return _etype;
}
@@ -307,18 +295,6 @@ public abstract class Hop implements ParseInfo {
invalid |=
!OptimizerUtils.isValidCPDimensions(in._dc.getRows(), in._dc.getCols());
return !invalid;
}
-
- public boolean hasMatrixInputWithDifferentBlocksizes() {
- for( Hop c : getInput() ) {
- if( c.getDataType()==DataType.MATRIX
- && getBlocksize() != c.getBlocksize() )
- {
- return true;
- }
- }
-
- return false;
- }
public void setRequiresReblock(boolean flag) {
_requiresReblock = flag;
diff --git a/src/main/java/org/apache/sysds/hops/IndexingOp.java
b/src/main/java/org/apache/sysds/hops/IndexingOp.java
index b3f80e441d..35215fa843 100644
--- a/src/main/java/org/apache/sysds/hops/IndexingOp.java
+++ b/src/main/java/org/apache/sysds/hops/IndexingOp.java
@@ -74,11 +74,6 @@ public class IndexingOp extends Hop
setColLowerEqualsUpper(passedColsLEU);
}
- @Override
- public void checkArity() {
- HopsException.check(_input.size() == 5, this, "should have 5
inputs but has %d inputs", _input.size());
- }
-
public boolean isRowLowerEqualsUpper(){
return _rowLowerEqualsUpper;
}
diff --git a/src/main/java/org/apache/sysds/hops/LeftIndexingOp.java
b/src/main/java/org/apache/sysds/hops/LeftIndexingOp.java
index 79d863adf9..e6fbc524fd 100644
--- a/src/main/java/org/apache/sysds/hops/LeftIndexingOp.java
+++ b/src/main/java/org/apache/sysds/hops/LeftIndexingOp.java
@@ -73,11 +73,6 @@ public class LeftIndexingOp extends Hop
setColLowerEqualsUpper(passedColsLEU);
}
- @Override
- public void checkArity() {
- HopsException.check(_input.size() == 6, this, "should have 6
inputs but has %d inputs", 6);
- }
-
public boolean isRowLowerEqualsUpper(){
return _rowLowerEqualsUpper;
}
diff --git a/src/main/java/org/apache/sysds/hops/LiteralOp.java
b/src/main/java/org/apache/sysds/hops/LiteralOp.java
index 1d2911f1fa..c2cb24f25c 100644
--- a/src/main/java/org/apache/sysds/hops/LiteralOp.java
+++ b/src/main/java/org/apache/sysds/hops/LiteralOp.java
@@ -70,11 +70,6 @@ public class LiteralOp extends Hop
value_boolean = that.value_boolean;
}
- @Override
- public void checkArity() {
- HopsException.check(_input.isEmpty(), this, "should have 0
inputs but has %d inputs", _input.size());
- }
-
@Override
public boolean isGPUEnabled() {
return false;
diff --git a/src/main/java/org/apache/sysds/hops/NaryOp.java
b/src/main/java/org/apache/sysds/hops/NaryOp.java
index 44fe8ff609..1659b0dbc5 100644
--- a/src/main/java/org/apache/sysds/hops/NaryOp.java
+++ b/src/main/java/org/apache/sysds/hops/NaryOp.java
@@ -67,10 +67,6 @@ public class NaryOp extends Hop {
refreshSizeInformation();
}
- /** MultipleOp may have any number of inputs. */
- @Override
- public void checkArity() {}
-
public OpOpN getOp() {
return _op;
}
diff --git a/src/main/java/org/apache/sysds/hops/ParameterizedBuiltinOp.java
b/src/main/java/org/apache/sysds/hops/ParameterizedBuiltinOp.java
index 3fa9afed3d..61a4b8b8f9 100644
--- a/src/main/java/org/apache/sysds/hops/ParameterizedBuiltinOp.java
+++ b/src/main/java/org/apache/sysds/hops/ParameterizedBuiltinOp.java
@@ -116,13 +116,6 @@ public class ParameterizedBuiltinOp extends
MultiThreadedHop {
refreshSizeInformation();
}
- @Override
- public void checkArity() {
- int sz = _input.size();
- int pz = _paramIndexMap.size();
- HopsException.check(sz == pz, this, "has %d inputs but %d
parameters", sz, pz);
- }
-
public HashMap<String, Integer> getParamIndexMap(){
return _paramIndexMap;
}
diff --git a/src/main/java/org/apache/sysds/hops/QuaternaryOp.java
b/src/main/java/org/apache/sysds/hops/QuaternaryOp.java
index 079cd33184..c2be949f37 100644
--- a/src/main/java/org/apache/sysds/hops/QuaternaryOp.java
+++ b/src/main/java/org/apache/sysds/hops/QuaternaryOp.java
@@ -155,12 +155,6 @@ public class QuaternaryOp extends MultiThreadedHop
inV.getParent().add(this);
}
- @Override
- public void checkArity() {
- HopsException.check(_input.size() == 3 || _input.size() == 4,
this,
- "should have arity 3 or 4 but has arity %d",
_input.size());
- }
-
public OpOp4 getOp(){
return _op;
}
diff --git a/src/main/java/org/apache/sysds/hops/ReorgOp.java
b/src/main/java/org/apache/sysds/hops/ReorgOp.java
index d7629eab10..576ccaa83a 100644
--- a/src/main/java/org/apache/sysds/hops/ReorgOp.java
+++ b/src/main/java/org/apache/sysds/hops/ReorgOp.java
@@ -81,27 +81,6 @@ public class ReorgOp extends MultiThreadedHop
refreshSizeInformation();
}
- @Override
- public void checkArity() {
- int sz = _input.size();
- switch( _op ) {
- case TRANS:
- case DIAG:
- case REV:
- HopsException.check(sz == 1, this, "should have
arity 1 for op %s but has arity %d", _op, sz);
- break;
- case ROLL:
- HopsException.check(sz == 2, this, "should have
arity 2 for op %s but has arity %d", _op, sz);
- break;
- case RESHAPE:
- case SORT:
- HopsException.check(sz == 5, this, "should have
arity 5 for op %s but has arity %d", _op, sz);
- break;
- default:
- throw new HopsException("Unsupported lops
construction for operation type '" + _op + "'.");
- }
- }
-
public ReOrgOp getOp() {
return _op;
}
diff --git a/src/main/java/org/apache/sysds/hops/TernaryOp.java
b/src/main/java/org/apache/sysds/hops/TernaryOp.java
index e6387b429c..87c99fc5c0 100644
--- a/src/main/java/org/apache/sysds/hops/TernaryOp.java
+++ b/src/main/java/org/apache/sysds/hops/TernaryOp.java
@@ -107,17 +107,6 @@ public class TernaryOp extends MultiThreadedHop
_dimInputsPresent = true;
}
- @Override
- public void checkArity() {
- int sz = _input.size();
- if (_dimInputsPresent) {
- // only CTABLE
- HopsException.check(sz == 5, this, "should have arity 5
for op %s but has arity %d", _op, sz);
- } else {
- HopsException.check(sz == 3, this, "should have arity 3
for op %s but has arity %d", _op, sz);
- }
- }
-
public OpOp3 getOp(){
return _op;
}
diff --git a/src/main/java/org/apache/sysds/hops/UnaryOp.java
b/src/main/java/org/apache/sysds/hops/UnaryOp.java
index 9c0e280644..2c0cd4a61b 100644
--- a/src/main/java/org/apache/sysds/hops/UnaryOp.java
+++ b/src/main/java/org/apache/sysds/hops/UnaryOp.java
@@ -74,11 +74,6 @@ public class UnaryOp extends MultiThreadedHop
refreshSizeInformation();
}
- @Override
- public void checkArity() {
- HopsException.check(_input.size() == 1, this, "should have
arity 1 but has arity %d", _input.size());
- }
-
// this is for OpOp1, e.g. A = -B (0-B); and a=!b
public OpOp1 getOp() {
return _op;
diff --git a/src/main/java/org/apache/sysds/hops/codegen/SpoofFusedOp.java
b/src/main/java/org/apache/sysds/hops/codegen/SpoofFusedOp.java
index d5bb1f0e72..b3b59adda1 100644
--- a/src/main/java/org/apache/sysds/hops/codegen/SpoofFusedOp.java
+++ b/src/main/java/org/apache/sysds/hops/codegen/SpoofFusedOp.java
@@ -78,9 +78,6 @@ public class SpoofFusedOp extends MultiThreadedHop
_genVarName = genVarName;
}
- @Override
- public void checkArity() {}
-
@Override
public boolean allowsAllExecTypes() {
return _distSupported;
diff --git a/src/main/java/org/apache/sysds/hops/ipa/FunctionCallGraph.java
b/src/main/java/org/apache/sysds/hops/ipa/FunctionCallGraph.java
index 177e13bb6c..bc223499b1 100644
--- a/src/main/java/org/apache/sysds/hops/ipa/FunctionCallGraph.java
+++ b/src/main/java/org/apache/sysds/hops/ipa/FunctionCallGraph.java
@@ -402,11 +402,7 @@ public class FunctionCallGraph
for (StatementBlock current : fs.getBody())
ret |= rConstructFunctionCallGraph(fkey,
current, fstack, lfset);
}
- else if (sb instanceof FunctionStatementBlock) {
- FunctionStatement fsb = (FunctionStatement)
sb.getStatement(0);
- for (StatementBlock current : fsb.getBody())
- ret |= rConstructFunctionCallGraph(fkey,
current, fstack, lfset);
- }
+ //FunctionStatementBlock handled on adding functions from basic
blocks
else {
// For generic StatementBlock
List<Hop> hopsDAG = sb.getHops();
diff --git a/src/main/java/org/apache/sysds/hops/rewrite/HopDagValidator.java
b/src/main/java/org/apache/sysds/hops/rewrite/HopDagValidator.java
index a8d77cf234..12f6d27d2e 100644
--- a/src/main/java/org/apache/sysds/hops/rewrite/HopDagValidator.java
+++ b/src/main/java/org/apache/sysds/hops/rewrite/HopDagValidator.java
@@ -121,9 +121,6 @@ public class HopDagValidator {
check(hop instanceof DataOp || hop instanceof
FunctionOp || hop instanceof LiteralOp, hop,
"is not a dataop/functionop/literal but
has no children");
- // check Hop has a legal arity (number of children)
- hop.checkArity();
-
// check Matrix data type Hops must have Double Value type
if (dt == DataType.MATRIX )
check(vt == ValueType.FP64 || vt == ValueType.INT64,
hop,
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/spark/functions/GetMIMBFromRow.java
b/src/main/java/org/apache/sysds/runtime/instructions/spark/functions/GetMIMBFromRow.java
deleted file mode 100644
index 481c136aaa..0000000000
---
a/src/main/java/org/apache/sysds/runtime/instructions/spark/functions/GetMIMBFromRow.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysds.runtime.instructions.spark.functions;
-
-import org.apache.spark.api.java.function.PairFunction;
-import org.apache.spark.sql.Row;
-import org.apache.sysds.runtime.matrix.data.MatrixBlock;
-import org.apache.sysds.runtime.matrix.data.MatrixIndexes;
-
-import scala.Tuple2;
-
-public class GetMIMBFromRow implements PairFunction<Row, MatrixIndexes,
MatrixBlock> {
-
- private static final long serialVersionUID = 2291741087248847581L;
-
- @Override
- public Tuple2<MatrixIndexes, MatrixBlock> call(Row row) throws
Exception {
- MatrixIndexes indx = (MatrixIndexes) row.apply(0);
- MatrixBlock blk = (MatrixBlock) row.apply(1);
- return new Tuple2<>(indx, blk);
- }
-
-}
diff --git
a/src/main/java/org/apache/sysds/runtime/matrix/data/CM_N_COVCell.java
b/src/main/java/org/apache/sysds/runtime/matrix/data/CM_N_COVCell.java
deleted file mode 100644
index a77ca3349d..0000000000
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/CM_N_COVCell.java
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysds.runtime.matrix.data;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.sysds.runtime.DMLRuntimeException;
-import org.apache.sysds.runtime.instructions.cp.CM_COV_Object;
-import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue;
-import org.apache.sysds.runtime.matrix.operators.AggregateOperator;
-import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator;
-import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
-import org.apache.sysds.runtime.matrix.operators.Operator;
-import org.apache.sysds.runtime.matrix.operators.ReorgOperator;
-import org.apache.sysds.runtime.matrix.operators.ScalarOperator;
-import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
-import org.apache.sysds.runtime.util.IndexRange;
-
-public class CM_N_COVCell extends MatrixValue
-{
- private CM_COV_Object cm=new CM_COV_Object();
-
- @Override
- public String toString() {
- return cm.toString();
- }
-
- @Override
- public MatrixValue binaryOperations(BinaryOperator op,
- MatrixValue thatValue, MatrixValue result) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public MatrixValue binaryOperationsInPlace(BinaryOperator op,
MatrixValue thatValue) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void copy(MatrixValue that, boolean sp) {
- throw new RuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void copy(MatrixValue that) {
- throw new RuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public long getNonZeros() {
- return 1;
- }
-
- @Override
- public int getNumColumns() {
- return 1;
- }
-
- @Override
- public int getNumRows() {
- return 1;
- }
-
- @Override
- public double get(int r, int c) {
- throw new RuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void incrementalAggregate(AggregateOperator aggOp,
- MatrixValue correction, MatrixValue newWithCorrection,
boolean deep) {
- throw new RuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void incrementalAggregate(AggregateOperator aggOp,
- MatrixValue newWithCorrection) {
- throw new RuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public boolean isInSparseFormat() {
- return false;
- }
-
- @Override
- public boolean isEmpty(){
- return false;
- }
-
- @Override
- public MatrixValue reorgOperations(ReorgOperator op, MatrixValue result,
- int startRow, int startColumn, int length) {
- throw new RuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void reset() {}
-
- @Override
- public void reset(int rl, int cl) {}
-
- @Override
- public void reset(int rl, int cl, boolean sp) {}
-
- @Override
- public void reset(int rl, int cl, boolean sp, long nnzs) {}
-
- @Override
- public void reset(int rl, int cl, double v) {}
-
- @Override
- public MatrixValue scalarOperations(ScalarOperator op, MatrixValue
result) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void set(int r, int c, double v) {
- throw new RuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public MatrixValue unaryOperations(UnaryOperator op, MatrixValue
result) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- cm.w=in.readDouble();
- cm.mean.read(in);
- cm.m2.read(in);
- cm.m3.read(in);
- cm.m4.read(in);
- cm.mean_v.read(in);
- cm.c2.read(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeDouble(cm.w);
- cm.mean.write(out);
- cm.m2.write(out);
- cm.m3.write(out);
- cm.m4.write(out);
- cm.mean_v.write(out);
- cm.c2.write(out);
- }
-
- @Override
- public int compareTo(Object o)
- {
- if(!(o instanceof CM_N_COVCell))
- return -1;
-
- CM_N_COVCell that=(CM_N_COVCell)o;
- return cm.compareTo(that.cm);
- }
-
- @Override
- public boolean equals(Object o)
- {
- if(!(o instanceof CM_N_COVCell))
- return false;
-
- CM_N_COVCell that=(CM_N_COVCell)o;
- return (cm==that.cm);
- }
-
- @Override
- public int hashCode() {
- throw new RuntimeException("hashCode() should never be called
on instances of this class.");
- }
-
- public CM_COV_Object getCM_N_COVObject()
- {
- return cm;
- }
-
- public void setCM_N_COVObject(double u, double v, double w)
- {
- cm.w=w;
- cm.mean.set(u,0);
- cm.mean_v.set(v, 0);
- cm.m2.set(0,0);
- cm.m3.set(0,0);
- cm.m4.set(0,0);
- cm.c2.set(0,0);
- }
- public void setCM_N_COVObject(CM_COV_Object that)
- {
- cm.set(that);
- }
-
- @Override
- public MatrixValue zeroOutOperations(MatrixValue result, IndexRange
range) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void ctableOperations(Operator op, MatrixValue that,
- MatrixValue that2, CTableMap resultMap, MatrixBlock
resultBlock) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void ctableOperations(Operator op, MatrixValue that,
- double scalarThat2, boolean ignoreZeros, CTableMap
resultMap, MatrixBlock resultBlock) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void ctableOperations(Operator op, double scalarThat,
- double scalarThat2, CTableMap resultMap, MatrixBlock
resultBlock) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void ctableOperations(Operator op, MatrixIndexes ix1, double
scalarThat, boolean left, int blen,
- CTableMap resultMap, MatrixBlock resultBlock) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void ctableOperations(Operator op, double scalarThat,
- MatrixValue that2, CTableMap resultMap, MatrixBlock
resultBlock) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void slice(ArrayList<IndexedMatrixValue> outlist,
- IndexRange range, int rowCut, int colCut, int blen, int
boundaryRlen, int boundaryClen) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public MatrixValue replaceOperations(MatrixValue result, double
pattern, double replacement) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public MatrixValue aggregateUnaryOperations(AggregateUnaryOperator op,
- MatrixValue result, int blen,
- MatrixIndexes indexesIn, boolean inCP) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-
- @Override
- public void append(MatrixValue valueIn2, ArrayList<IndexedMatrixValue>
outlist,
- int blen, boolean cbind, boolean m2IsLast, int
nextNCol) {
- throw new DMLRuntimeException("operation not supported for
CM_N_COVCell");
- }
-}
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixCell.java
b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixCell.java
index 4782dba879..0f9ea2621e 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixCell.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixCell.java
@@ -27,8 +27,6 @@ import java.io.Serializable;
import java.util.ArrayList;
import org.apache.sysds.runtime.DMLRuntimeException;
-import org.apache.sysds.runtime.functionobjects.CTable;
-import org.apache.sysds.runtime.functionobjects.ReduceDiag;
import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue;
import org.apache.sysds.runtime.matrix.operators.AggregateOperator;
import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator;
@@ -45,21 +43,10 @@ public class MatrixCell extends MatrixValue implements
Serializable
protected double value;
- public MatrixCell()
- {
+ public MatrixCell() {
value=0;
}
- public MatrixCell(MatrixCell that)
- {
- this.value=that.value;
- }
-
- public MatrixCell(MatrixValue that) {
- if(that instanceof MatrixCell)
- this.value=((MatrixCell)that).value;
- }
-
public MatrixCell(double v) {
value=v;
}
@@ -167,20 +154,12 @@ public class MatrixCell extends MatrixValue implements
Serializable
@Override
public MatrixValue binaryOperations(BinaryOperator op,
MatrixValue thatValue, MatrixValue result) {
- MatrixCell c2=checkType(thatValue);
- MatrixCell c3=checkType(result);
- if(c3==null)
- c3=new MatrixCell();
- c3.setValue(op.fn.execute(this.getValue(), c2.getValue()));
- return c3;
+ throw new UnsupportedOperationException();
}
@Override
- public MatrixValue binaryOperationsInPlace(BinaryOperator op,
- MatrixValue thatValue) {
- MatrixCell c2=checkType(thatValue);
- setValue(op.fn.execute(this.getValue(), c2.getValue()));
- return this;
+ public MatrixValue binaryOperationsInPlace(BinaryOperator op,
MatrixValue thatValue) {
+ throw new UnsupportedOperationException();
}
public void denseScalarOperationsInPlace(ScalarOperator op) {
@@ -190,11 +169,7 @@ public class MatrixCell extends MatrixValue implements
Serializable
@Override
public MatrixValue reorgOperations(ReorgOperator op, MatrixValue result,
int startRow, int startColumn, int length) {
- MatrixCell c3=checkType(result);
- if(c3==null)
- c3=new MatrixCell();
- c3.setValue(getValue());
- return c3;
+ throw new UnsupportedOperationException();
}
@Override
@@ -245,127 +220,70 @@ public class MatrixCell extends MatrixValue implements
Serializable
@Override
public void incrementalAggregate(AggregateOperator aggOp,
MatrixValue correction, MatrixValue newWithCorrection,
boolean deep) {
- throw new DMLRuntimeException("MatrixCell.incrementalAggregate
should never be called");
+ throw new UnsupportedOperationException();
}
@Override
public MatrixValue zeroOutOperations(MatrixValue result, IndexRange
range) {
- if(range.rowStart!=0 || range.rowEnd!=0 || range.colStart!=0 ||
range.colEnd!=0)
- throw new DMLRuntimeException("wrong range: "+range+"
for matrixCell");
- MatrixCell c3=checkType(result);
- c3.setValue(value);
- return c3;
+ throw new UnsupportedOperationException();
}
@Override
public void incrementalAggregate(AggregateOperator aggOp,
MatrixValue newWithCorrection) {
- throw new DMLRuntimeException("MatrixCell.incrementalAggregate
should never be called");
+ throw new UnsupportedOperationException();
}
@Override
public void ctableOperations(Operator op, MatrixValue that,
MatrixValue that2, CTableMap resultMap, MatrixBlock
resultBlock) {
- MatrixCell c2=checkType(that);
- MatrixCell c3=checkType(that2);
- CTable ctable = CTable.getCTableFnObject();
- if ( resultMap != null)
- ctable.execute(this.value, c2.value, c3.value, false,
resultMap);
- else
- ctable.execute(this.value, c2.value, c3.value, false,
resultBlock);
+ throw new UnsupportedOperationException();
}
@Override
public void ctableOperations(Operator op, MatrixValue that, double
scalarThat2, boolean ignoreZeros,
CTableMap ctableResult, MatrixBlock ctableResultBlock) {
- MatrixCell c2=checkType(that);
- CTable ctable = CTable.getCTableFnObject();
- if ( ctableResult != null)
- ctable.execute(this.value, c2.value, scalarThat2,
ignoreZeros, ctableResult);
- else
- ctable.execute(this.value, c2.value, scalarThat2,
ignoreZeros, ctableResultBlock);
+ throw new UnsupportedOperationException();
}
@Override
public void ctableOperations(Operator op, double scalarThat,
double scalarThat2, CTableMap resultMap, MatrixBlock
resultBlock) {
- CTable ctable = CTable.getCTableFnObject();
- if ( resultMap != null)
- ctable.execute(this.value, scalarThat, scalarThat2,
false, resultMap);
- else
- ctable.execute(this.value, scalarThat, scalarThat2,
false, resultBlock);
+ throw new UnsupportedOperationException();
}
@Override
public void ctableOperations(Operator op, MatrixIndexes ix1, double
scalarThat, boolean left, int blen,
CTableMap resultMap, MatrixBlock resultBlock) {
- //ctable expand (column vector to ctable)
- CTable ctable = CTable.getCTableFnObject();
- if ( resultMap != null ) {
- if( left )
- ctable.execute(ix1.getRowIndex(), this.value,
scalarThat, false, resultMap);
- else
- ctable.execute(this.value, ix1.getRowIndex(),
scalarThat, false, resultMap);
- }
- else {
- if( left )
- ctable.execute(ix1.getRowIndex(), this.value,
scalarThat, false, resultBlock);
- else
- ctable.execute(this.value, ix1.getRowIndex(),
scalarThat, false, resultBlock);
- }
+ throw new UnsupportedOperationException();
}
@Override
public void ctableOperations(Operator op, double scalarThat,
MatrixValue that2, CTableMap resultMap, MatrixBlock
resultBlock) {
- MatrixCell c3=checkType(that2);
- CTable ctable = CTable.getCTableFnObject();
- if ( resultMap != null)
- ctable.execute(this.value, scalarThat, c3.value, false,
resultMap);
- else
- ctable.execute(this.value, scalarThat, c3.value, false,
resultBlock);
-
+ throw new UnsupportedOperationException();
}
@Override
public void slice(ArrayList<IndexedMatrixValue> outlist,
IndexRange range, int rowCut, int colCut, int blen, int
boundaryRlen, int boundaryClen) {
- ((MatrixCell)outlist.get(0).getValue()).setValue(this.value);
+ throw new UnsupportedOperationException();
}
@Override
public MatrixValue replaceOperations(MatrixValue result, double
pattern, double replacement) {
- MatrixCell out = checkType(result);
- if( value == pattern || (Double.isNaN(pattern) &&
Double.isNaN(value)) )
- out.value = replacement;
- else
- out.value = value;
- return out;
+ throw new UnsupportedOperationException();
}
@Override
public MatrixValue aggregateUnaryOperations(AggregateUnaryOperator op,
MatrixValue result, int blen,
MatrixIndexes indexesIn, boolean inCP) {
- MatrixCell c3 = checkType(result);
- if(c3 == null)
- c3 = new MatrixCell();
-
- if(op.indexFn instanceof ReduceDiag) {
- if(indexesIn.getRowIndex() ==
indexesIn.getColumnIndex())
- c3.setValue(getValue());
- else
- c3.setValue(0);
- }
- else
- c3.setValue(getValue());
- return c3;
+ throw new UnsupportedOperationException();
}
@Override
public void append(MatrixValue valueIn2, ArrayList<IndexedMatrixValue>
outlist,
int blen, boolean cbind, boolean m2IsLast, int
nextNCol) {
- ((MatrixCell)outlist.get(0).getValue()).setValue(this.value);
- MatrixCell c2=checkType(valueIn2);
- ((MatrixCell)outlist.get(1).getValue()).setValue(c2.getValue());
+ throw new UnsupportedOperationException();
}
}
diff --git
a/src/main/java/org/apache/sysds/runtime/matrix/data/TextToBinaryCellConverter.java
b/src/main/java/org/apache/sysds/runtime/matrix/data/TextToBinaryCellConverter.java
deleted file mode 100644
index 525d459f75..0000000000
---
a/src/main/java/org/apache/sysds/runtime/matrix/data/TextToBinaryCellConverter.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysds.runtime.matrix.data;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.sysds.runtime.util.FastStringTokenizer;
-
-
-public class TextToBinaryCellConverter
-implements Converter<LongWritable, Text, MatrixIndexes, MatrixCell>
-{
- private MatrixIndexes indexes = new MatrixIndexes();
- private MatrixCell value = new MatrixCell();
- private Pair<MatrixIndexes, MatrixCell> pair = new Pair<>(indexes,
value);
- private FastStringTokenizer st = new FastStringTokenizer(' ');
- private boolean hasValue = false;
- private boolean toIgnore = false;
-
- @Override
- public void convert(LongWritable k1, Text v1)
- {
- String str = v1.toString();
-
- //handle support for matrix market format
- if(str.startsWith("%")) {
- if(str.startsWith("%%"))
- toIgnore=true;
- hasValue=false;
- return;
- }
- else if(toIgnore) {
- toIgnore=false;
- hasValue=false;
- return;
- }
-
- //reset the tokenizer
- st.reset( str );
-
- //convert text to matrix cell
- indexes.setIndexes( st.nextLong(), st.nextLong() );
- if( indexes.getRowIndex() == 0 || indexes.getColumnIndex() == 0
) {
- hasValue = false;
- return;
- }
- value.setValue( st.nextDouble() );
- hasValue = true;
- }
-
- @Override
- public boolean hasNext() {
- return hasValue;
- }
-
- @Override
- public Pair<MatrixIndexes, MatrixCell> next() {
- if(!hasValue)
- return null;
- hasValue=false;
- return pair;
- }
-
- @Override
- public void setBlockSize(int rl, int cl) {
- //do nothing
- }
-}
diff --git
a/src/main/java/org/apache/sysds/runtime/matrix/data/WeightedPair.java
b/src/main/java/org/apache/sysds/runtime/matrix/data/WeightedPair.java
deleted file mode 100644
index fd4bb86db5..0000000000
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/WeightedPair.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysds.runtime.matrix.data;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.sysds.runtime.DMLRuntimeException;
-
-public class WeightedPair extends WeightedCell
-{
-
- private static final long serialVersionUID = 8772815876289553196L;
-
- private double other=0;
-
- @Override
- public String toString() {
- return value+", "+other+": "+weight;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- value=in.readDouble();
- other=in.readDouble();
- weight=in.readDouble();
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeDouble(value);
- out.writeDouble(other);
- out.writeDouble(weight);
- }
-
- private static WeightedPair checkType(MatrixValue cell) {
- if( cell!=null && !(cell instanceof WeightedPair))
- throw new DMLRuntimeException("the Matrix Value is not
WeightedPair!");
- return (WeightedPair) cell;
- }
-
- @Override
- public void copy(MatrixValue that){
- WeightedPair c2;
- try {
- c2 = checkType(that);
- } catch (DMLRuntimeException e) {
- throw new RuntimeException(e);
- }
- value=c2.getValue();
- other=c2.getOtherValue();
- weight=c2.getWeight();
- }
-
- public double getOtherValue() {
- return other;
- }
-
- public void setOtherValue(double ov)
- {
- other=ov;
- }
-
- @Override
- public int compareTo(Object o)
- {
- if( !(o instanceof WeightedPair) )
- return -1;
-
- WeightedPair that = (WeightedPair)o;
- if(this.value!=that.value)
- return Double.compare(this.value, that.value);
- else if(this.other!=that.other)
- return Double.compare(this.other, that.other);
- else if(this.weight!=that.weight)
- return Double.compare(this.weight, that.weight);
- else return 0;
- }
-
- @Override
- public boolean equals(Object o)
- {
- if( !(o instanceof WeightedPair) )
- return false;
-
- WeightedPair that = (WeightedPair)o;
- return (value==that.value && other==that.other && weight ==
that.weight);
- }
-
- @Override
- public int hashCode() {
- throw new RuntimeException("hashCode() should never be called
on instances of this class.");
- }
-}
diff --git
a/src/main/java/org/apache/sysds/runtime/matrix/operators/ReIndexOperator.java
b/src/main/java/org/apache/sysds/runtime/matrix/operators/ReIndexOperator.java
deleted file mode 100644
index 6cba69c4b6..0000000000
---
a/src/main/java/org/apache/sysds/runtime/matrix/operators/ReIndexOperator.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysds.runtime.matrix.operators;
-
-public class ReIndexOperator extends Operator
-{
- private static final long serialVersionUID = 8603367674384408297L;
-
- public ReIndexOperator() {
- super(true);
- }
-}
diff --git
a/src/main/java/org/apache/sysds/runtime/meta/DataCharacteristics.java
b/src/main/java/org/apache/sysds/runtime/meta/DataCharacteristics.java
index 073434ad81..d714c2115b 100644
--- a/src/main/java/org/apache/sysds/runtime/meta/DataCharacteristics.java
+++ b/src/main/java/org/apache/sysds/runtime/meta/DataCharacteristics.java
@@ -19,10 +19,6 @@
package org.apache.sysds.runtime.meta;
-import org.apache.sysds.runtime.matrix.operators.AggregateBinaryOperator;
-import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator;
-import org.apache.sysds.runtime.matrix.operators.ReorgOperator;
-
import java.io.Serializable;
public abstract class DataCharacteristics implements Serializable {
@@ -122,18 +118,6 @@ public abstract class DataCharacteristics implements
Serializable {
public abstract boolean mightHaveEmptyBlocks();
- public static void reorg(DataCharacteristics dim, ReorgOperator op,
DataCharacteristics dimOut) {
- op.fn.computeDimension(dim, dimOut);
- }
-
- public static void aggregateUnary(DataCharacteristics dim,
AggregateUnaryOperator op, DataCharacteristics dimOut) {
- op.indexFn.computeDimension(dim, dimOut);
- }
-
- public static void aggregateBinary(DataCharacteristics dim1,
DataCharacteristics dim2, AggregateBinaryOperator op, DataCharacteristics
dimOut) {
- dimOut.set(dim1.getRows(), dim2.getCols(), dim1.getBlocksize());
- }
-
public abstract boolean equalDims(Object anObject);
@Override
diff --git
a/src/test/java/org/apache/sysds/test/component/utils/HopDagValidationTest.java
b/src/test/java/org/apache/sysds/test/component/utils/HopDagValidationTest.java
new file mode 100644
index 0000000000..c192b1b083
--- /dev/null
+++
b/src/test/java/org/apache/sysds/test/component/utils/HopDagValidationTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.utils;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysds.api.DMLScript;
+import org.apache.sysds.conf.ConfigurationManager;
+import org.apache.sysds.conf.DMLConfig;
+import org.apache.sysds.hops.rewrite.HopDagValidator;
+import org.apache.sysds.hops.rewrite.RewriteAlgebraicSimplificationStatic;
+import org.apache.sysds.parser.DMLProgram;
+import org.apache.sysds.parser.DMLTranslator;
+import org.apache.sysds.parser.ParserFactory;
+import org.apache.sysds.parser.ParserWrapper;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+
+public class HopDagValidationTest extends AutomatedTestBase
+{
+ private static final String TEST_DIR = "component/parfor/";
+ private static final String HOME = SCRIPT_DIR + TEST_DIR;
+ private static final String TEST_CLASS_DIR = TEST_DIR +
HopDagValidationTest.class.getSimpleName() + "/";
+
+ @Override
+ public void setUp() {}
+
+ @Test
+ public void testDependencyAnalysis1() { runTest("parfor1.dml"); }
+
+ @Test
+ public void testDependencyAnalysis3() { runTest("parfor3.dml"); }
+
+ @Test
+ public void testDependencyAnalysis4() { runTest("parfor4.dml"); }
+
+ @Test
+ public void testDependencyAnalysis6() { runTest("parfor6.dml"); }
+
+ @Test
+ public void testDependencyAnalysis7() { runTest("parfor7.dml"); }
+
+
+ private void runTest( String scriptFilename ) {
+ int index = scriptFilename.lastIndexOf(".dml");
+ String testName = scriptFilename.substring(0, index > 0 ? index
: scriptFilename.length());
+ TestConfiguration testConfig = new
TestConfiguration(TEST_CLASS_DIR, testName, new String[] {});
+ addTestConfiguration(testName, testConfig);
+ loadTestConfiguration(testConfig);
+
+ try {
+ DMLConfig conf = new
DMLConfig(getCurConfigFile().getPath());
+ ConfigurationManager.setLocalConfig(conf);
+
+ //read script
+ String dmlScriptString = DMLScript.readDMLScript(true,
HOME + scriptFilename);
+
+ //parsing and dependency analysis
+ ParserWrapper parser = ParserFactory.createParser();
+ DMLProgram prog =
parser.parse(DMLScript.DML_FILE_PATH_ANTLR_PARSER, dmlScriptString, new
HashMap<>());
+ DMLTranslator dmlt = new DMLTranslator(prog);
+ dmlt.validateParseTree(prog);
+
HopDagValidator.validateHopDag(prog.getStatementBlocks().get(0).getHops(),
+ new RewriteAlgebraicSimplificationStatic());
+ }
+ catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+}
