Repository: kylin Updated Branches: refs/heads/master 3bca0bfe4 -> 6845a358c
KYLIN-1694 make multiply coefficient configurable when estimating cuboid size Signed-off-by: lidongsjtu <lid...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6845a358 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6845a358 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6845a358 Branch: refs/heads/master Commit: 6845a358c2b7d56ee453bcb8d769d0fcb9b609ea Parents: 3bca0bf Author: kangkaisen <kangkai...@live.com> Authored: Mon May 16 14:49:26 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Fri May 27 13:54:50 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/common/KylinConfigBase.java | 8 ++++++++ .../apache/kylin/engine/mr/common/CubeStatsReader.java | 11 +++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/6845a358/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b2294b0..497f29d 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -409,6 +409,14 @@ abstract public class KylinConfigBase implements Serializable { return Long.parseLong(getOptional("kylin.job.step.timeout", String.valueOf(2 * 60 * 60))); } + public double getJobCuboidSizeRatio(){ + return Double.parseDouble(getOptional("kylin.job.cuboid.size.ratio", "0.25")); + } + + public double getJobCuboidSizeMemHungryRatio(){ + return Double.parseDouble(getOptional("kylin.job.cuboid.size.memhungry.ratio", "0.05")); + } + public String getCubeAlgorithm() { return getOptional("kylin.cube.algorithm", "auto"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/6845a358/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java index 940172b..8973a99 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java @@ -180,6 +180,7 @@ public class CubeStatsReader { private static double estimateCuboidStorageSize(CubeSegment cubeSegment, long cuboidId, long rowCount, long baseCuboidId, List<Integer> rowKeyColumnLength) { int bytesLength = cubeSegment.getRowKeyPreambleSize(); + KylinConfig kylinConf = cubeSegment.getConfig(); long mask = Long.highestOneBit(baseCuboidId); long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId); @@ -204,11 +205,13 @@ public class CubeStatsReader { double ret = 1.0 * bytesLength * rowCount / (1024L * 1024L); if (isMemoryHungry) { - logger.info("Cube is memory hungry, storage size estimation multiply 0.05"); - ret *= 0.05; + double cuboidSizeMemHungryRatio = kylinConf.getJobCuboidSizeMemHungryRatio(); + logger.info("Cube is memory hungry, storage size estimation multiply " + cuboidSizeMemHungryRatio); + ret *= cuboidSizeMemHungryRatio; } else { - logger.info("Cube is not memory hungry, storage size estimation multiply 0.25"); - ret *= 0.25; + double cuboidSizeRatio = kylinConf.getJobCuboidSizeRatio(); + logger.info("Cube is not memory hungry, storage size estimation multiply " + cuboidSizeRatio); + ret *= cuboidSizeRatio; } logger.info("Cuboid " + cuboidId + " has " + rowCount + " rows, each row size is " + bytesLength + " bytes." + " Total size is " + ret + "M."); return ret;
