This is an automated email from the ASF dual-hosted git repository.
sebwrede pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 5ed7368 [MINOR] Move IPAPassRewriteFederatedPlan to IPA Package
5ed7368 is described below
commit 5ed73681ddad939b526027395d017473ccc8a1f1
Author: sebwrede <[email protected]>
AuthorDate: Mon Nov 8 12:44:38 2021 +0100
[MINOR] Move IPAPassRewriteFederatedPlan to IPA Package
The IPAPassRewriteFederatedPlan is moved to the IPA package where it
belongs.
Additionally, the code is auto-formatted.
Closes #1445.
---
.../hops/rewrite/IPAPassRewriteFederatedPlan.html | 2 +-
.../class-use/IPAPassRewriteFederatedPlan.html | 8 +-
.../IPAPassRewriteFederatedPlan.java | 124 +++++++++++----------
.../sysds/hops/ipa/InterProceduralAnalysis.java | 1 -
4 files changed, 71 insertions(+), 64 deletions(-)
diff --git
a/docs/api/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.html
b/docs/api/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.html
index b7039b9..21bb921 100644
---
a/docs/api/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.html
+++
b/docs/api/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.html
@@ -103,7 +103,7 @@ var activeTableTab = "activeTableTab";
<li><a href="../../../../../org/apache/sysds/hops/ipa/IPAPass.html"
title="class in
org.apache.sysds.hops.ipa">org.apache.sysds.hops.ipa.IPAPass</a></li>
<li>
<ul class="inheritance">
-<li>org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan</li>
+<li>org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan</li>
</ul>
</li>
</ul>
diff --git
a/docs/api/java/org/apache/sysds/hops/rewrite/class-use/IPAPassRewriteFederatedPlan.html
b/docs/api/java/org/apache/sysds/hops/rewrite/class-use/IPAPassRewriteFederatedPlan.html
index 3218937..c31c389 100644
---
a/docs/api/java/org/apache/sysds/hops/rewrite/class-use/IPAPassRewriteFederatedPlan.html
+++
b/docs/api/java/org/apache/sysds/hops/rewrite/class-use/IPAPassRewriteFederatedPlan.html
@@ -4,7 +4,7 @@
<head>
<!-- Generated by javadoc -->
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<title>Uses of Class org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan
(SystemDS 2.3.0-SNAPSHOT API)</title>
+<title>Uses of Class org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan
(SystemDS 2.3.0-SNAPSHOT API)</title>
<link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css"
title="Style">
<script type="text/javascript" src="../../../../../../script.js"></script>
</head>
@@ -12,7 +12,7 @@
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
- parent.document.title="Uses of Class
org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan (SystemDS
2.3.0-SNAPSHOT API)";
+ parent.document.title="Uses of Class
org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan (SystemDS 2.3.0-SNAPSHOT
API)";
}
}
catch(err) {
@@ -70,9 +70,9 @@
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<div class="header">
-<h2 title="Uses of Class
org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan" class="title">Uses
of Class<br>org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan</h2>
+<h2 title="Uses of Class
org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan" class="title">Uses of
Class<br>org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan</h2>
</div>
-<div class="classUseContainer">No usage of
org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan</div>
+<div class="classUseContainer">No usage of
org.apache.sysds.hops.ipa.IPAPassRewriteFederatedPlan</div>
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a name="navbar.bottom">
<!-- -->
diff --git
a/src/main/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.java
b/src/main/java/org/apache/sysds/hops/ipa/IPAPassRewriteFederatedPlan.java
similarity index 79%
rename from
src/main/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.java
rename to
src/main/java/org/apache/sysds/hops/ipa/IPAPassRewriteFederatedPlan.java
index 2a28399..8c8df49 100644
---
a/src/main/java/org/apache/sysds/hops/rewrite/IPAPassRewriteFederatedPlan.java
+++ b/src/main/java/org/apache/sysds/hops/ipa/IPAPassRewriteFederatedPlan.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.sysds.hops.rewrite;
+package org.apache.sysds.hops.ipa;
import org.apache.sysds.api.DMLException;
import org.apache.sysds.hops.AggBinaryOp;
@@ -30,9 +30,6 @@ import org.apache.sysds.hops.OptimizerUtils;
import org.apache.sysds.hops.ReorgOp;
import org.apache.sysds.hops.TernaryOp;
import org.apache.sysds.hops.cost.HopRel;
-import org.apache.sysds.hops.ipa.FunctionCallGraph;
-import org.apache.sysds.hops.ipa.FunctionCallSizeInfo;
-import org.apache.sysds.hops.ipa.IPAPass;
import org.apache.sysds.parser.DMLProgram;
import org.apache.sysds.parser.ForStatement;
import org.apache.sysds.parser.ForStatementBlock;
@@ -69,8 +66,7 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
* @param fgraph function call graph
* @return true if federated compilation is activated.
*/
- @Override
- public boolean isApplicable(FunctionCallGraph fgraph) {
+ @Override public boolean isApplicable(FunctionCallGraph fgraph) {
return OptimizerUtils.FEDERATED_COMPILATION;
}
@@ -83,8 +79,8 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
* @param fcallSizes function call size infos
* @return false since the function call graph never has to be rebuilt
*/
- @Override
- public boolean rewriteProgram(DMLProgram prog, FunctionCallGraph
fgraph, FunctionCallSizeInfo fcallSizes) {
+ @Override public boolean rewriteProgram(DMLProgram prog,
FunctionCallGraph fgraph,
+ FunctionCallSizeInfo fcallSizes) {
rewriteStatementBlocks(prog, prog.getStatementBlocks());
return false;
}
@@ -95,12 +91,12 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
* The method calls the contained statement blocks recursively.
*
* @param prog dml program
- * @param sbs list of statement blocks
+ * @param sbs list of statement blocks
* @return list of statement blocks with the federated output value
updated for each hop
*/
public ArrayList<StatementBlock> rewriteStatementBlocks(DMLProgram
prog, List<StatementBlock> sbs) {
ArrayList<StatementBlock> rewrittenStmBlocks = new
ArrayList<>();
- for ( StatementBlock stmBlock : sbs )
+ for(StatementBlock stmBlock : sbs)
rewrittenStmBlocks.addAll(rewriteStatementBlock(prog,
stmBlock));
return rewrittenStmBlocks;
}
@@ -111,19 +107,19 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
* The method calls the contained statement blocks recursively.
*
* @param prog dml program
- * @param sb statement block
+ * @param sb statement block
* @return list of statement blocks with the federated output value
updated for each hop
*/
public ArrayList<StatementBlock> rewriteStatementBlock(DMLProgram prog,
StatementBlock sb) {
- if ( sb instanceof WhileStatementBlock)
+ if(sb instanceof WhileStatementBlock)
return rewriteWhileStatementBlock(prog,
(WhileStatementBlock) sb);
- else if ( sb instanceof IfStatementBlock)
+ else if(sb instanceof IfStatementBlock)
return rewriteIfStatementBlock(prog, (IfStatementBlock)
sb);
- else if ( sb instanceof ForStatementBlock){
+ else if(sb instanceof ForStatementBlock) {
// This also includes ParForStatementBlocks
return rewriteForStatementBlock(prog,
(ForStatementBlock) sb);
}
- else if ( sb instanceof FunctionStatementBlock)
+ else if(sb instanceof FunctionStatementBlock)
return rewriteFunctionStatementBlock(prog,
(FunctionStatementBlock) sb);
else {
// StatementBlock type (no subclass)
@@ -131,19 +127,19 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
}
}
- private ArrayList<StatementBlock> rewriteWhileStatementBlock(DMLProgram
prog, WhileStatementBlock whileSB){
+ private ArrayList<StatementBlock> rewriteWhileStatementBlock(DMLProgram
prog, WhileStatementBlock whileSB) {
Hop whilePredicateHop = whileSB.getPredicateHops();
selectFederatedExecutionPlan(whilePredicateHop);
- for ( Statement stm : whileSB.getStatements() ){
+ for(Statement stm : whileSB.getStatements()) {
WhileStatement whileStm = (WhileStatement) stm;
whileStm.setBody(rewriteStatementBlocks(prog,
whileStm.getBody()));
}
return new ArrayList<>(Collections.singletonList(whileSB));
}
- private ArrayList<StatementBlock> rewriteIfStatementBlock(DMLProgram
prog, IfStatementBlock ifSB){
+ private ArrayList<StatementBlock> rewriteIfStatementBlock(DMLProgram
prog, IfStatementBlock ifSB) {
selectFederatedExecutionPlan(ifSB.getPredicateHops());
- for ( Statement statement : ifSB.getStatements() ){
+ for(Statement statement : ifSB.getStatements()) {
IfStatement ifStatement = (IfStatement) statement;
ifStatement.setIfBody(rewriteStatementBlocks(prog,
ifStatement.getIfBody()));
ifStatement.setElseBody(rewriteStatementBlocks(prog,
ifStatement.getElseBody()));
@@ -151,34 +147,35 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
return new ArrayList<>(Collections.singletonList(ifSB));
}
- private ArrayList<StatementBlock> rewriteForStatementBlock(DMLProgram
prog, ForStatementBlock forSB){
+ private ArrayList<StatementBlock> rewriteForStatementBlock(DMLProgram
prog, ForStatementBlock forSB) {
selectFederatedExecutionPlan(forSB.getFromHops());
selectFederatedExecutionPlan(forSB.getToHops());
selectFederatedExecutionPlan(forSB.getIncrementHops());
- for ( Statement statement : forSB.getStatements() ){
- ForStatement forStatement = ((ForStatement)statement);
+ for(Statement statement : forSB.getStatements()) {
+ ForStatement forStatement = ((ForStatement) statement);
forStatement.setBody(rewriteStatementBlocks(prog,
forStatement.getBody()));
}
return new ArrayList<>(Collections.singletonList(forSB));
}
- private ArrayList<StatementBlock>
rewriteFunctionStatementBlock(DMLProgram prog, FunctionStatementBlock funcSB){
- for ( Statement statement : funcSB.getStatements() ){
+ private ArrayList<StatementBlock>
rewriteFunctionStatementBlock(DMLProgram prog, FunctionStatementBlock funcSB) {
+ for(Statement statement : funcSB.getStatements()) {
FunctionStatement funcStm = (FunctionStatement)
statement;
funcStm.setBody(rewriteStatementBlocks(prog,
funcStm.getBody()));
}
return new ArrayList<>(Collections.singletonList(funcSB));
}
- private ArrayList<StatementBlock>
rewriteDefaultStatementBlock(DMLProgram prog, StatementBlock sb){
- if ( sb.getHops() != null && !sb.getHops().isEmpty() ){
- for ( Hop sbHop : sb.getHops() ){
- if ( sbHop instanceof FunctionOp ){
+ private ArrayList<StatementBlock>
rewriteDefaultStatementBlock(DMLProgram prog, StatementBlock sb) {
+ if(sb.getHops() != null && !sb.getHops().isEmpty()) {
+ for(Hop sbHop : sb.getHops()) {
+ if(sbHop instanceof FunctionOp) {
String funcName = ((FunctionOp)
sbHop).getFunctionName();
FunctionStatementBlock sbFuncBlock =
prog.getBuiltinFunctionDictionary().getFunction(funcName);
rewriteStatementBlock(prog,
sbFuncBlock);
}
- else selectFederatedExecutionPlan(sbHop);
+ else
+ selectFederatedExecutionPlan(sbHop);
}
}
return new ArrayList<>(Collections.singletonList(sb));
@@ -188,40 +185,45 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
* Sets FederatedOutput field of all hops in DAG starting from given
root.
* The FederatedOutput chosen for root is the minimum cost HopRel found
in memo table for the given root.
* The FederatedOutput values chosen for the inputs to the root are
chosen based on the input dependencies.
+ *
* @param root hop for which FederatedOutput needs to be set
*/
- private void setFinalFedout(Hop root){
- HopRel optimalRootHopRel =
hopRelMemo.get(root.getHopID()).stream().min(Comparator.comparingDouble(HopRel::getCost))
+ private void setFinalFedout(Hop root) {
+ HopRel optimalRootHopRel =
hopRelMemo.get(root.getHopID()).stream()
+ .min(Comparator.comparingDouble(HopRel::getCost))
.orElseThrow(() -> new DMLException("Hop root " + root
+ " has no feasible federated output alternatives"));
setFinalFedout(root, optimalRootHopRel);
}
/**
* Update the FederatedOutput value and cost based on information
stored in given rootHopRel.
- * @param root hop for which FederatedOutput is set
+ *
+ * @param root hop for which FederatedOutput is set
* @param rootHopRel from which FederatedOutput value and cost is
retrieved
*/
- private void setFinalFedout(Hop root, HopRel rootHopRel){
+ private void setFinalFedout(Hop root, HopRel rootHopRel) {
updateFederatedOutput(root, rootHopRel);
visitInputDependency(rootHopRel);
}
/**
* Sets FederatedOutput value for each of the inputs of rootHopRel
+ *
* @param rootHopRel which has its input values updated
*/
- private void visitInputDependency(HopRel rootHopRel){
+ private void visitInputDependency(HopRel rootHopRel) {
List<HopRel> hopRelInputs = rootHopRel.getInputDependency();
- for ( HopRel input : hopRelInputs )
+ for(HopRel input : hopRelInputs)
setFinalFedout(input.getHopRef(), input);
}
/**
* Updates FederatedOutput value and cost estimate based on
updateHopRel values.
- * @param root which has its values updated
+ *
+ * @param root which has its values updated
* @param updateHopRel from which the values are retrieved
*/
- private void updateFederatedOutput(Hop root, HopRel updateHopRel){
+ private void updateFederatedOutput(Hop root, HopRel updateHopRel) {
root.setFederatedOutput(updateHopRel.getFederatedOutput());
root.setFederatedCost(updateHopRel.getCostObject());
}
@@ -229,6 +231,7 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
/**
* Select federated execution plan for every Hop in the DAG starting
from given roots.
* The cost estimates of the hops are also updated when FederatedOutput
is updated in the hops.
+ *
* @param roots starting point for going through the Hop DAG to update
the FederatedOutput fields.
*/
@SuppressWarnings("unused")
@@ -239,57 +242,61 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
/**
* Select federated execution plan for every Hop in the DAG starting
from given root.
+ *
* @param root starting point for going through the Hop DAG to update
the federatedOutput fields
*/
- private void selectFederatedExecutionPlan(Hop root){
+ private void selectFederatedExecutionPlan(Hop root) {
visitFedPlanHop(root);
setFinalFedout(root);
}
/**
* Go through the Hop DAG and set the FederatedOutput field and cost
estimate for each Hop from leaf to given currentHop.
+ *
* @param currentHop the Hop from which the DAG is visited
*/
- private void visitFedPlanHop(Hop currentHop){
+ private void visitFedPlanHop(Hop currentHop) {
// If the currentHop is in the hopRelMemo table, it means that
it has been visited
- if ( hopRelMemo.containsKey(currentHop.getHopID()) )
+ if(hopRelMemo.containsKey(currentHop.getHopID()))
return;
// If the currentHop has input, then the input should be
visited depth-first
- if ( currentHop.getInput() != null &&
currentHop.getInput().size() > 0 ){
- for ( Hop input : currentHop.getInput() )
+ if(currentHop.getInput() != null &&
currentHop.getInput().size() > 0) {
+ for(Hop input : currentHop.getInput())
visitFedPlanHop(input);
}
// Put FOUT, LOUT, and None HopRels into the memo table
ArrayList<HopRel> hopRels = new ArrayList<>();
- if ( isFedInstSupportedHop(currentHop) ){
- for ( FEDInstruction.FederatedOutput fedoutValue :
FEDInstruction.FederatedOutput.values() )
- if ( isFedOutSupported(currentHop, fedoutValue)
)
- hopRels.add(new
HopRel(currentHop,fedoutValue, hopRelMemo));
+ if(isFedInstSupportedHop(currentHop)) {
+ for(FEDInstruction.FederatedOutput fedoutValue :
FEDInstruction.FederatedOutput.values())
+ if(isFedOutSupported(currentHop, fedoutValue))
+ hopRels.add(new HopRel(currentHop,
fedoutValue, hopRelMemo));
}
- if ( hopRels.isEmpty() )
+ if(hopRels.isEmpty())
hopRels.add(new HopRel(currentHop,
FEDInstruction.FederatedOutput.NONE, hopRelMemo));
hopRelMemo.put(currentHop.getHopID(), hopRels);
}
/**
* Checks if the instructions related to the given hop supports
FOUT/LOUT processing.
+ *
* @param hop to check for federated support
* @return true if federated instructions related to hop supports
FOUT/LOUT processing
*/
- private boolean isFedInstSupportedHop(Hop hop){
+ private boolean isFedInstSupportedHop(Hop hop) {
// The following operations are supported given that the above
conditions have not returned already
- return ( hop instanceof AggBinaryOp || hop instanceof BinaryOp
|| hop instanceof ReorgOp
+ return (hop instanceof AggBinaryOp || hop instanceof BinaryOp
|| hop instanceof ReorgOp
|| hop instanceof AggUnaryOp || hop instanceof
TernaryOp || hop instanceof DataOp);
}
/**
* Checks if the associatedHop supports the given federated output
value.
+ *
* @param associatedHop to check support of
- * @param fedOut federated output value
+ * @param fedOut federated output value
* @return true if associatedHop supports fedOut
*/
- private boolean isFedOutSupported(Hop associatedHop,
FEDInstruction.FederatedOutput fedOut){
- switch(fedOut){
+ private boolean isFedOutSupported(Hop associatedHop,
FEDInstruction.FederatedOutput fedOut) {
+ switch(fedOut) {
case FOUT:
return isFOUTSupported(associatedHop);
case LOUT:
@@ -303,17 +310,17 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
/**
* Checks to see if the associatedHop supports FOUT.
+ *
* @param associatedHop for which FOUT support is checked
* @return true if FOUT is supported by the associatedHop
*/
- private boolean isFOUTSupported(Hop associatedHop){
+ private boolean isFOUTSupported(Hop associatedHop) {
// If the output of AggUnaryOp is a scalar, the operation
cannot be FOUT
- if ( associatedHop instanceof AggUnaryOp &&
associatedHop.isScalar() )
+ if(associatedHop instanceof AggUnaryOp &&
associatedHop.isScalar())
return false;
// It can only be FOUT if at least one of the inputs are FOUT,
except if it is a federated DataOp
- if ( associatedHop.getInput().stream().noneMatch(
- input ->
hopRelMemo.get(input.getHopID()).stream().anyMatch(HopRel::hasFederatedOutput) )
- && !associatedHop.isFederatedDataOp() )
+ if(associatedHop.getInput().stream().noneMatch(input ->
hopRelMemo.get(input.getHopID()).stream()
+ .anyMatch(HopRel::hasFederatedOutput)) &&
!associatedHop.isFederatedDataOp())
return false;
return true;
}
@@ -321,10 +328,11 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
/**
* Checks to see if the associatedHop supports LOUT.
* It supports LOUT if the output has no privacy constraints.
+ *
* @param associatedHop for which LOUT support is checked.
* @return true if LOUT is supported by the associatedHop
*/
- private boolean isLOUTSupported(Hop associatedHop){
+ private boolean isLOUTSupported(Hop associatedHop) {
return associatedHop.getPrivacy() == null ||
!associatedHop.getPrivacy().hasConstraints();
}
}
diff --git
a/src/main/java/org/apache/sysds/hops/ipa/InterProceduralAnalysis.java
b/src/main/java/org/apache/sysds/hops/ipa/InterProceduralAnalysis.java
index b0597eb..20644fa 100644
--- a/src/main/java/org/apache/sysds/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysds/hops/ipa/InterProceduralAnalysis.java
@@ -34,7 +34,6 @@ import org.apache.sysds.hops.HopsException;
import org.apache.sysds.hops.LiteralOp;
import org.apache.sysds.hops.OptimizerUtils;
import org.apache.sysds.hops.recompile.Recompiler;
-import org.apache.sysds.hops.rewrite.IPAPassRewriteFederatedPlan;
import org.apache.sysds.parser.DMLProgram;
import org.apache.sysds.parser.DMLTranslator;
import org.apache.sysds.parser.DataIdentifier;
