Repository: systemml Updated Branches: refs/heads/master 9a1f64b42 -> 3b87c2ba9
[MINOR] Fine tuning spark checkpoint data size thresholds Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/3b87c2ba Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/3b87c2ba Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/3b87c2ba Branch: refs/heads/master Commit: 3b87c2ba9d77ffa3d901eae38de9c1157994d74e Parents: 9a1f64b Author: Matthias Boehm <mboe...@gmail.com> Authored: Wed Dec 12 13:53:23 2018 +0100 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Wed Dec 12 13:53:23 2018 +0100 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/OptimizerUtils.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/3b87c2ba/src/main/java/org/apache/sysml/hops/OptimizerUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java index a43abb3..e6a25d2 100644 --- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java +++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java @@ -914,7 +914,11 @@ public class OptimizerUtils * @return true if the given matrix characteristics exceed threshold */ public static boolean exceedsCachingThreshold(long dim2, double outMem) { - return !(dim2 > 1 && outMem < getLocalMemBudget() + //NOTE: We heuristically cache matrices that are close to or larger + //than the local memory budget. The different relative fractions + //according to number of columns is reflecting common operations + //(e.g., two inputs/one output for binary vector operations) + return !(dim2 > 1 && outMem < getLocalMemBudget()/2 || dim2 == 1 && outMem < getLocalMemBudget()/3); }
