[FLINK-4094] [core] Add a warning to only use off heap memory with pre-allocation=true
This closes #2336 Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/119364c6 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/119364c6 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/119364c6 Branch: refs/heads/master Commit: 119364c6e38aedf7e6ebf8b2e84632a0a3697377 Parents: 8d4a64d Author: Ramkrishna <ramkrishna.s.vasude...@intel.com> Authored: Fri Aug 5 11:25:07 2016 +0530 Committer: Stephan Ewen <se...@apache.org> Committed: Fri Aug 5 16:03:58 2016 +0200 ---------------------------------------------------------------------- docs/setup/config.md | 2 +- .../java/org/apache/flink/runtime/memory/MemoryManager.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/119364c6/docs/setup/config.md ---------------------------------------------------------------------- diff --git a/docs/setup/config.md b/docs/setup/config.md index f4faac5..5baccfa 100644 --- a/docs/setup/config.md +++ b/docs/setup/config.md @@ -83,7 +83,7 @@ The default fraction for managed memory can be adjusted using the `taskmanager.m - `taskmanager.memory.segment-size`: The size of memory buffers used by the memory manager and the network stack in bytes (DEFAULT: 32768 (= 32 KiBytes)). -- `taskmanager.memory.preallocate`: Can be either of `true` or `false`. Specifies whether task managers should allocate all managed memory when starting up. (DEFAULT: false) +- `taskmanager.memory.preallocate`: Can be either of `true` or `false`. Specifies whether task managers should allocate all managed memory when starting up. (DEFAULT: false). When `taskmanager.memory.off-heap` is set to `true`, then it is advised that this configuration is also set to `true`. If this configuration is set to `false` cleaning up of the allocated offheap memory happens only when the configured JVM parameter MaxDirectMemorySize is reached by triggering a full GC. ### Memory and Performance Debugging http://git-wip-us.apache.org/repos/asf/flink/blob/119364c6/flink-runtime/src/main/java/org/apache/flink/runtime/memory/MemoryManager.java ---------------------------------------------------------------------- diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/memory/MemoryManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/memory/MemoryManager.java index 3ade753..81d410c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/memory/MemoryManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/memory/MemoryManager.java @@ -34,6 +34,8 @@ import org.apache.flink.core.memory.HybridMemorySegment; import org.apache.flink.core.memory.MemorySegment; import org.apache.flink.core.memory.MemoryType; import org.apache.flink.util.MathUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The memory manager governs the memory that Flink uses for sorting, hashing, and caching. Memory @@ -53,6 +55,7 @@ import org.apache.flink.util.MathUtils; */ public class MemoryManager { + private static final Logger LOG = LoggerFactory.getLogger(MemoryManager.class); /** The default memory page size. Currently set to 32 KiBytes. */ public static final int DEFAULT_PAGE_SIZE = 32 * 1024; @@ -163,6 +166,10 @@ public class MemoryManager { this.memoryPool = new HeapMemoryPool(memToAllocate, pageSize); break; case OFF_HEAP: + if(!preAllocateMemory) { + LOG.warn("It is advisable to set 'taskmanager.memory.preallocate' to true when" + + " the memory type 'taskmanager.memory.off-heap' is set to true."); + } this.memoryPool = new HybridOffHeapMemoryPool(memToAllocate, pageSize); break; default:
