Repository: kylin Updated Branches: refs/heads/master 41014d6a5 -> 22e6969cb
KYLIN-1591 enabling different block size setting for small column families Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/22e6969c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/22e6969c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/22e6969c Branch: refs/heads/master Commit: 22e6969cb30f2146b23a9c81701b8a39dba15fd4 Parents: 41014d6 Author: Hongbin Ma <mahong...@apache.org> Authored: Mon Apr 18 15:40:05 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Mon Apr 18 15:40:54 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/common/KylinConfigBase.java | 10 +++++++--- .../apache/kylin/cube/model/HBaseColumnFamilyDesc.java | 13 +++++++++++++ .../java/org/apache/kylin/gridtable/GTScanRequest.java | 3 ++- .../kylin/storage/hbase/steps/CubeHTableUtil.java | 9 ++++++--- 4 files changed, 28 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 51aa8aa..4c7995b 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -489,7 +489,7 @@ abstract public class KylinConfigBase implements Serializable { } public int getQueryScanFuzzyKeyMax() { - return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200")); + return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200")); } public int getQueryStorageVisitScanRangeMax() { @@ -558,8 +558,12 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.hbase.default.encoding", "FAST_DIFF"); } - public String getHbaseDefaultBlockSize() { - return getOptional("kylin.hbase.default.block.size", "4194304"); + public int getHbaseDefaultBlockSize() { + return Integer.valueOf(getOptional("kylin.hbase.default.block.size", "1048576")); + } + + public int getHbaseSmallFamilyBlockSize() { + return Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536")); } public boolean isHiveKeepFlatTable() { http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java index 5db2a9b..c5b2e19 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java @@ -20,6 +20,8 @@ package org.apache.kylin.cube.model; import java.util.Arrays; +import org.apache.kylin.metadata.model.MeasureDesc; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; @@ -55,4 +57,15 @@ public class HBaseColumnFamilyDesc { return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + Arrays.toString(columns) + "]"; } + public boolean isMemoryHungry() { + for (HBaseColumnDesc hBaseColumnDesc : columns) { + for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) { + if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) { + return true; + } + } + } + return false; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index 5681057..bc478c9 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -161,7 +161,8 @@ public class GTScanRequest { } if (!doAggr) {//Skip reading this section if you're not profiling! - int scanned = lookAndForget(result); + int scanned = result.getScannedRowCount(); + lookAndForget(result); return new EmptyGTScanner(scanned); } http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java index 182cffe..e7e980a 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java @@ -93,6 +93,12 @@ public class CubeHTableUtil { String hbaseDefaultCC = kylinConfig.getHbaseDefaultCompressionCodec().toLowerCase(); + if (cfDesc.isMemoryHungry()) { + cf.setBlocksize(kylinConfig.getHbaseDefaultBlockSize()); + } else { + cf.setBlocksize(kylinConfig.getHbaseSmallFamilyBlockSize()); + } + switch (hbaseDefaultCC) { case "snappy": { logger.info("hbase will use snappy to compress data"); @@ -121,9 +127,6 @@ public class CubeHTableUtil { } } - int blockSize = Integer.valueOf(kylinConfig.getHbaseDefaultBlockSize()); - cf.setBlocksize(blockSize); - try { String encodingStr = kylinConfig.getHbaseDefaultEncoding(); DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr);
