Repository: kylin Updated Branches: refs/heads/2.0-rc 192ecee55 -> e01e4b602
minor tweaks to avoid OOM in InMemCubeBuilder Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e01e4b60 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e01e4b60 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e01e4b60 Branch: refs/heads/2.0-rc Commit: e01e4b6024593b453448c22026614f24d91dcfd5 Parents: 192ecee Author: Li, Yang <[email protected]> Authored: Fri Dec 11 17:05:33 2015 +0800 Committer: Li, Yang <[email protected]> Committed: Fri Dec 11 17:05:33 2015 +0800 ---------------------------------------------------------------------- .../kylin/cube/inmemcubing/DoggedCubeBuilder.java | 12 ++++++------ .../kylin/cube/inmemcubing/InMemCubeBuilder.java | 15 +++++---------- .../java/org/apache/kylin/cube/model/CubeDesc.java | 9 +++++++++ 3 files changed, 20 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e01e4b60/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java index 7fe2122..a8e8c94 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java @@ -56,6 +56,10 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder { public DoggedCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<?>> dictionaryMap) { super(cubeDesc, dictionaryMap); + + // check memory more often if a single row is big + if (cubeDesc.hasMemoryHungryCountDistinctMeasures()) + unitRows /= 10; } public void setSplitRowThreshold(int rowThreshold) { @@ -241,12 +245,8 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder { } // wait cuboid build done - while (last.isAlive()) { - if (last.builder.isAllCuboidDone()) { - break; - } - Thread.sleep(1000); - } + last.join(); + } catch (InterruptedException e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e01e4b60/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java index e9d940a..19acfda 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java @@ -61,7 +61,7 @@ import com.google.common.collect.Lists; public class InMemCubeBuilder extends AbstractInMemCubeBuilder { private static Logger logger = LoggerFactory.getLogger(InMemCubeBuilder.class); - static final double BASE_CUBOID_CACHE_OVERSIZE_FACTOR = 0.1; + static final double BASE_CUBOID_CACHE_OVERSIZE_FACTOR = 0.15; private final CuboidScheduler cuboidScheduler; private final long baseCuboidId; @@ -100,8 +100,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]); } - - private GridTable newGridTableByCuboidID(long cuboidID) throws IOException { GTInfo info = CubeGridTable.newGTInfo(cubeDesc, cuboidID, dictionaryMap); @@ -114,7 +112,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { return gridTable; } - @Override public void build(BlockingQueue<List<String>> input, ICuboidWriter output) throws IOException { ConcurrentNavigableMap<Long, CuboidResult> result = build(input); @@ -301,14 +298,14 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { // GC to be precise on memory left Runtime.getRuntime().gc(); try { - Thread.sleep(1000); + Thread.sleep(2500); } catch (InterruptedException e) { logger.error("", e); } // GC again to be precise on memory left Runtime.getRuntime().gc(); try { - Thread.sleep(1000); + Thread.sleep(2500); } catch (InterruptedException e) { logger.error("", e); } @@ -345,7 +342,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, true); long startTime = System.currentTimeMillis(); - logger.info("Calculating cuboid " + baseCuboidId); + logger.info("Calculating base cuboid " + baseCuboidId + ", system avail " + mbBefore + " MB"); int count = 0; for (GTRecord r : aggregationScanner) { @@ -508,9 +505,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { this.info = info; this.input = input; this.record = new GTRecord(info); - this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc, - InMemCubeBuilderUtils.createTopNLiteralColDictionaryMap(cubeDesc, intermediateTableDesc, dictionaryMap), - info); + this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc, InMemCubeBuilderUtils.createTopNLiteralColDictionaryMap(cubeDesc, intermediateTableDesc, dictionaryMap), info); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/e01e4b60/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index ef563ed..17dc650 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -738,6 +738,15 @@ public class CubeDesc extends RootPersistentEntity { return false; } + public boolean hasMemoryHungryCountDistinctMeasures() { + for (MeasureDesc measure : measures) { + if (measure.getFunction().isCountDistinct()) { + return true; + } + } + return false; + } + public long getRetentionRange() { return retentionRange; }
