Repository: incubator-kylin Updated Branches: refs/heads/2.x-staging 2f865b80e -> 101ef86ac
KYLIN-942 only check mem use at query time Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/101ef86a Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/101ef86a Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/101ef86a Branch: refs/heads/2.x-staging Commit: 101ef86ac8a411d290d4bde80eeb29d91afcfa95 Parents: 2f865b8 Author: honma <ho...@ebay.com> Authored: Mon Nov 2 18:52:36 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Mon Nov 2 18:52:58 2015 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/util/MemoryBudgetController.java | 2 +- .../org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java | 2 +- .../java/org/apache/kylin/gridtable/GTAggregateScanner.java | 5 +++-- .../src/main/java/org/apache/kylin/gridtable/GTScanRequest.java | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java index 2c26666..dc49b17 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java @@ -57,7 +57,7 @@ public class MemoryBudgetController { public static final MemoryBudgetController ZERO_BUDGET = new MemoryBudgetController(0); public static final int ONE_MB = 1024 * 1024; - public static final int ONE_GB = 1024 * 1024 * 1024; + public static final long ONE_GB = 1024 * 1024 * 1024; private static final Logger logger = LoggerFactory.getLogger(MemoryBudgetController.class); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java index d7aa50a..a179d70 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java @@ -343,7 +343,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null); - GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, false); + GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, true); long startTime = System.currentTimeMillis(); logger.info("Calculating cuboid " + baseCuboidId); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java index 35c1210..cd2881e 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java @@ -142,8 +142,9 @@ public class GTAggregateScanner implements IGTScanner { void aggregate(GTRecord r) { if (enableMemCheck && (++aggregatedRowCount % 100000 == 0)) { - if (estimatedMemSize() > MemoryBudgetController.ONE_GB) { - throw new RuntimeException("AggregationCache exceed 1GB"); + long estimated = estimatedMemSize(); + if (estimated > 10 * MemoryBudgetController.ONE_GB) { + throw new RuntimeException("AggregationCache exceed 10GB, estimated size is: " + estimated); } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index ba997ec..2c284c9 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -118,7 +118,7 @@ public class GTScanRequest { } public IGTScanner decorateScanner(IGTScanner scanner) throws IOException { - return decorateScanner(scanner, true, true, true); + return decorateScanner(scanner, true, true, false);//by default do not check mem } /**