KYLIN-1367 Use by-layer cubing algorithm if there is memory hungry measure Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6671a0db Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6671a0db Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6671a0db
Branch: refs/heads/2.x-staging Commit: 6671a0db9103826fc74b577efa5e5e64728b2934 Parents: d10fb6a Author: shaofengshi <shaofeng...@apache.org> Authored: Mon Jan 25 17:20:55 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Mon Jan 25 17:20:55 2016 +0800 ---------------------------------------------------------------------- .../engine/mr/steps/SaveStatisticsStep.java | 30 ++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/6671a0db/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java index 396955b..9314b88 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java @@ -39,6 +39,7 @@ import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableContext; import org.apache.kylin.job.execution.ExecuteResult; +import org.apache.kylin.metadata.model.MeasureDesc; /** * Save the cube segment statistic to Kylin metadata store @@ -95,15 +96,28 @@ public class SaveStatisticsStep extends AbstractExecutable { alg = AlgorithmEnum.INMEM; } else if (AlgorithmEnum.LAYER.name().equalsIgnoreCase(algPref)) { alg = AlgorithmEnum.LAYER; - } else if ("random".equalsIgnoreCase(algPref)) { // for testing - alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER; - } else { // the default - double threshold = kylinConf.getCubeAlgorithmAutoThreshold(); - double mapperOverlapRatio = new CubeStatsReader(seg, kylinConf).getMapperOverlapRatioOfFirstBuild(); - logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + threshold); - alg = mapperOverlapRatio < threshold ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER; - } + } else { + boolean memoryHungry = false; + for (MeasureDesc measure : seg.getCubeDesc().getMeasures()) { + if (measure.getFunction().getMeasureType().isMemoryHungry()) { + logger.info("This cube has memory-hungry measure " + measure.getFunction().getExpression()); + memoryHungry = true; + break; + } + } + + if (memoryHungry == true) { + alg = AlgorithmEnum.LAYER; + } else if ("random".equalsIgnoreCase(algPref)) { // for testing + alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER; + } else { // the default + double threshold = kylinConf.getCubeAlgorithmAutoThreshold(); + double mapperOverlapRatio = new CubeStatsReader(seg, kylinConf).getMapperOverlapRatioOfFirstBuild(); + logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + threshold); + alg = mapperOverlapRatio < threshold ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER; + } + } logger.info("The cube algorithm for " + seg + " is " + alg); CubingJob cubingJob = (CubingJob) executableManager.getJob(getCubingJobId());
