KYLIN-1190 Make memory budget per query configurable Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e8e2d27a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e8e2d27a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e8e2d27a Branch: refs/heads/2.x-staging Commit: e8e2d27ab82d6565615a27d4a441f019a5b5aafa Parents: 6b20569 Author: lidongsjtu <don...@ebay.com> Authored: Thu Dec 3 09:50:19 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Fri Dec 4 16:29:07 2015 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/kylin/common/KylinConfig.java | 4 ++++ .../kylin/storage/hbase/cube/v1/CubeStorageQuery.java | 12 ++++++++---- .../kylin/storage/hbase/cube/v2/CubeStorageQuery.java | 11 +++++++---- 3 files changed, 19 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index b08df5f..d2a94c5 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -492,6 +492,10 @@ public class KylinConfig implements Serializable { return Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", String.valueOf(10 * 1024))); } + public long getQueryMemBudget() { + return Long.parseLong(this.getOptional("kylin.query.mem.budget", String.valueOf(3L * 1024 * 1024 * 1024))); + } + public boolean isQuerySecureEnabled() { return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java index 4d34943..2fa0490 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java @@ -82,7 +82,6 @@ public class CubeStorageQuery implements ICachableStorageQuery { private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class); private static final int MERGE_KEYRANGE_THRESHOLD = 100; - private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G private final CubeInstance cubeInstance; private final CubeDesc cubeDesc; @@ -732,8 +731,13 @@ public class CubeStorageQuery implements ICachableStorageQuery { } } - long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst; - context.setThreshold((int) rowEst); + long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst; + if (rowEst > 0) { + logger.info("Memory budget is set to: " + rowEst); + context.setThreshold((int) rowEst); + } else { + logger.info("Memory budget is not set."); + } } private void setLimit(TupleFilter filter, StorageContext context) { @@ -760,7 +764,7 @@ public class CubeStorageQuery implements ICachableStorageQuery { topnLiteralCol = func.getTopNLiteralColumn(); } } - + // if TopN is not involved if (topnFunc == null) return; http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java index 0c8c3bd..59ed99b 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java @@ -41,8 +41,6 @@ public class CubeStorageQuery implements ICachableStorageQuery { private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class); - private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G - private final CubeInstance cubeInstance; private final CubeDesc cubeDesc; @@ -362,8 +360,13 @@ public class CubeStorageQuery implements ICachableStorageQuery { rowSizeEst += func.getReturnDataType().getStorageBytesEstimate(); } - long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst; - context.setThreshold((int) rowEst); + long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst; + if (rowEst > 0) { + logger.info("Memory budget is set to: " + rowEst); + context.setThreshold((int) rowEst); + } else { + logger.info("Memory budget is not set."); + } } private void setLimit(TupleFilter filter, StorageContext context) {