Repository: systemml Updated Branches: refs/heads/master 3d4c21b01 -> d45b84863
[SYSTEMML-1791] Performance frame cache release (avoid scans) This patch removes unnecessary frame cache release overhead for JMLC deployments. On cache release, we update the size of currently pinned objects, which is only used in the hybrid spark execution mode. For frame with string schema this creates unnecessary overhead because for string column this is not a meta data operation but requires a full scan over the individual strings to determine their lengths. We now disable this in forced CP execution mode, which significantly improved performance in the context of column indexing, which are meanwhile pure meta data operations. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d45b8486 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d45b8486 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d45b8486 Branch: refs/heads/master Commit: d45b84863e889f6fb2cacc31722147ce8409bf22 Parents: 3d4c21b Author: Matthias Boehm <mboe...@gmail.com> Authored: Thu Jul 20 15:39:24 2017 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Thu Jul 20 16:21:47 2017 -0700 ---------------------------------------------------------------------- .../runtime/controlprogram/caching/CacheableData.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/d45b8486/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java index 4084eb9..a9f604f 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.conf.ConfigurationManager; +import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.DMLRuntimeException; @@ -595,7 +596,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data //update meta data refreshMetaData(); } - + //compact empty in-memory block _data.compactEmptyBlock(); @@ -1242,11 +1243,11 @@ public abstract class CacheableData<T extends CacheBlock> extends Data } protected void updateStatusPinned(boolean add) { - if( _data != null ) { //data should never be null - long size = sizePinned.get(); - size += (add ? 1 : -1) * _data.getInMemorySize(); - sizePinned.set( Math.max(size,0) ); - } + if( _data == null || !OptimizerUtils.isHybridExecutionMode() ) + return; //avoid size computation for string frames + long size = sizePinned.get(); + size += (add ? 1 : -1) * _data.getInMemorySize(); + sizePinned.set( Math.max(size,0) ); } protected long getPinnedSize() {