DRILL-5669: Add a configurable option for minimum memory allocation to buffered ops
closes #879 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/6b0e8378 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/6b0e8378 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/6b0e8378 Branch: refs/heads/master Commit: 6b0e8378cc198c926f5b1cf95a07fd4c9fc092de Parents: 35a7ad0 Author: Boaz Ben-Zvi <boazben-zvi@BBenZvi-E754-MBP13.local> Authored: Mon Jul 17 14:21:29 2017 -0700 Committer: Arina Ielchiieva <arina.yelchiy...@gmail.com> Committed: Fri Jul 21 16:32:26 2017 +0300 ---------------------------------------------------------------------- .../src/main/java/org/apache/drill/exec/ExecConstants.java | 9 +++++++++ .../drill/exec/server/options/SystemOptionManager.java | 1 + .../apache/drill/exec/util/MemoryAllocationUtilities.java | 8 ++++++-- 3 files changed, 16 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/6b0e8378/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index 5b82d1f..97cb321 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -343,6 +343,15 @@ public interface ExecConstants { MAX_QUERY_MEMORY_PER_NODE_KEY, 1024 * 1024, Long.MAX_VALUE, 2 * 1024 * 1024 * 1024L); /** + * Minimum memory alocated to each buffered operator instance. + * <p/> + * DEFAULT: 40 MB + */ + String MIN_MEMORY_PER_BUFFERED_OP_KEY = "planner.memory.min_memory_per_buffered_op"; + LongValidator MIN_MEMORY_PER_BUFFERED_OP = new RangeLongValidator( + MIN_MEMORY_PER_BUFFERED_OP_KEY, 1024 * 1024, Long.MAX_VALUE, 40 * 1024 * 1024L); + + /** * Extra query memory per node for non-blocking operators. * NOTE: This option is currently used only for memory estimation. * <p/> http://git-wip-us.apache.org/repos/asf/drill/blob/6b0e8378/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index c2a4d65..3392a21 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -144,6 +144,7 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana ExecConstants.EARLY_LIMIT0_OPT, ExecConstants.ENABLE_MEMORY_ESTIMATION, ExecConstants.MAX_QUERY_MEMORY_PER_NODE, + ExecConstants.MIN_MEMORY_PER_BUFFERED_OP, ExecConstants.NON_BLOCKING_OPERATORS_MEMORY, ExecConstants.HASH_JOIN_TABLE_FACTOR, ExecConstants.HASH_AGG_TABLE_FACTOR, http://git-wip-us.apache.org/repos/asf/drill/blob/6b0e8378/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java index 79b49e4..4580222 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java @@ -66,11 +66,15 @@ public class MemoryAllocationUtilities { final long maxOperatorAlloc = maxAllocPerNode / (bufferedOpList.size() * maxWidthPerNode); logger.debug("Max buffered operator alloc: {}", maxOperatorAlloc); + // User configurable option to allow forcing minimum memory. + // Ensure that the buffered ops receive the minimum memory needed to make progress. + // Without this, the math might work out to allocate too little memory. + final long opMinMem = queryContext.getOptions().getOption(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP_KEY).num_val; + for(final PhysicalOperator op : bufferedOpList) { - // Ensure that the sort receives the minimum memory needed to make progress. - // Without this, the math might work out to allocate too little memory. long alloc = Math.max(maxOperatorAlloc, op.getInitialAllocation()); + alloc = Math.max(alloc, opMinMem); op.setMaxAllocation(alloc); } }
