[31/31] incubator-kylin git commit: KYLIN-942 only check mem use at query time

[shaofengshi]

KYLIN-942 only check mem use at query time


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/91057748
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/91057748
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/91057748

Branch: refs/heads/KYLIN-1112
Commit: 91057748ce02c5ebb750f062253c927536300343
Parents: 42f1254
Author: honma 
Authored: Mon Nov 2 18:52:36 2015 +0800
Committer: shaofengshi 
Committed: Thu Nov 5 09:53:17 2015 +0800

--
 .../org/apache/kylin/common/util/MemoryBudgetController.java| 2 +-
 .../org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java | 2 +-
 .../java/org/apache/kylin/gridtable/GTAggregateScanner.java | 5 +++--
 .../src/main/java/org/apache/kylin/gridtable/GTScanRequest.java | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91057748/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
 
b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index 2c2..dc49b17 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -57,7 +57,7 @@ public class MemoryBudgetController {
 
 public static final MemoryBudgetController ZERO_BUDGET = new 
MemoryBudgetController(0);
 public static final int ONE_MB = 1024 * 1024;
-public static final int ONE_GB = 1024 * 1024 * 1024;
+public static final long ONE_GB = 1024 * 1024 * 1024;
 
 private static final Logger logger = 
LoggerFactory.getLogger(MemoryBudgetController.class);
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91057748/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index d7aa50a..a179d70 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -343,7 +343,7 @@ public class InMemCubeBuilder extends 
AbstractInMemCubeBuilder {
 
 Pair dimensionMetricsBitSet = 
InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, 
measureCount);
 GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, 
dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), 
metricsAggrFuncs, null);
-GTAggregateScanner aggregationScanner = new 
GTAggregateScanner(baseInput, req, false);
+GTAggregateScanner aggregationScanner = new 
GTAggregateScanner(baseInput, req, true);
 
 long startTime = System.currentTimeMillis();
 logger.info("Calculating cuboid " + baseCuboidId);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91057748/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 35c1210..cd2881e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -142,8 +142,9 @@ public class GTAggregateScanner implements IGTScanner {
 
 void aggregate(GTRecord r) {
 if (enableMemCheck && (++aggregatedRowCount % 10 == 0)) {
-if (estimatedMemSize() > MemoryBudgetController.ONE_GB) {
-throw new RuntimeException("AggregationCache exceed 1GB");
+long estimated = estimatedMemSize();
+if (estimated > 10 * MemoryBudgetController.ONE_GB) {
+throw new RuntimeException("AggregationCache exceed 10GB, 
estimated size is: " + estimated);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/91057748/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 

incubator-kylin git commit: KYLIN-942 only check mem use at query time

[mahongbin]

Repository: incubator-kylin
Updated Branches:
  refs/heads/2.x-staging 2f865b80e -> 101ef86ac


KYLIN-942 only check mem use at query time


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/101ef86a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/101ef86a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/101ef86a

Branch: refs/heads/2.x-staging
Commit: 101ef86ac8a411d290d4bde80eeb29d91afcfa95
Parents: 2f865b8
Author: honma 
Authored: Mon Nov 2 18:52:36 2015 +0800
Committer: honma 
Committed: Mon Nov 2 18:52:58 2015 +0800

--
 .../org/apache/kylin/common/util/MemoryBudgetController.java| 2 +-
 .../org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java | 2 +-
 .../java/org/apache/kylin/gridtable/GTAggregateScanner.java | 5 +++--
 .../src/main/java/org/apache/kylin/gridtable/GTScanRequest.java | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
 
b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index 2c2..dc49b17 100644
--- 
a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ 
b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -57,7 +57,7 @@ public class MemoryBudgetController {
 
 public static final MemoryBudgetController ZERO_BUDGET = new 
MemoryBudgetController(0);
 public static final int ONE_MB = 1024 * 1024;
-public static final int ONE_GB = 1024 * 1024 * 1024;
+public static final long ONE_GB = 1024 * 1024 * 1024;
 
 private static final Logger logger = 
LoggerFactory.getLogger(MemoryBudgetController.class);
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index d7aa50a..a179d70 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -343,7 +343,7 @@ public class InMemCubeBuilder extends 
AbstractInMemCubeBuilder {
 
 Pair dimensionMetricsBitSet = 
InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, 
measureCount);
 GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, 
dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), 
metricsAggrFuncs, null);
-GTAggregateScanner aggregationScanner = new 
GTAggregateScanner(baseInput, req, false);
+GTAggregateScanner aggregationScanner = new 
GTAggregateScanner(baseInput, req, true);
 
 long startTime = System.currentTimeMillis();
 logger.info("Calculating cuboid " + baseCuboidId);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 35c1210..cd2881e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -142,8 +142,9 @@ public class GTAggregateScanner implements IGTScanner {
 
 void aggregate(GTRecord r) {
 if (enableMemCheck && (++aggregatedRowCount % 10 == 0)) {
-if (estimatedMemSize() > MemoryBudgetController.ONE_GB) {
-throw new RuntimeException("AggregationCache exceed 1GB");
+long estimated = estimatedMemSize();
+if (estimated > 10 * MemoryBudgetController.ONE_GB) {
+throw new RuntimeException("AggregationCache exceed 10GB, 
estimated size is: " + estimated);
 }
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git