KYLIN-1323 correct the default region cut config Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/576ec1b0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/576ec1b0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/576ec1b0
Branch: refs/heads/1.5.x-HBase1.x Commit: 576ec1b0bf7b311ed3823685919c6385e7ce0518 Parents: 60848ce Author: shaofengshi <shaofeng...@apache.org> Authored: Tue May 3 19:09:17 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Tue May 3 19:11:15 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfigBase.java | 4 ++-- .../test_kylin_ii_left_join_desc.json | 2 +- .../kylin/storage/hbase/steps/CreateHTableJob.java | 15 +++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/576ec1b0/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 05211b1..97245ce 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -411,7 +411,7 @@ abstract public class KylinConfigBase implements Serializable { } public float getHBaseHFileSizeGB() { - return Float.parseFloat(getOptional("kylin.hbase.hfile.size.gb", "5.0")); + return Float.parseFloat(getOptional("kylin.hbase.hfile.size.gb", "2.0")); } public int getScanThreshold() { @@ -511,7 +511,7 @@ abstract public class KylinConfigBase implements Serializable { * @return */ public float getKylinHBaseRegionCut() { - return Float.valueOf(getOptional("kylin.hbase.region.cut", "1.0")); + return Float.valueOf(getOptional("kylin.hbase.region.cut", "5.0")); } public int getHBaseScanMaxResultSize() { http://git-wip-us.apache.org/repos/asf/kylin/blob/576ec1b0/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json index d397392..2f152ad 100644 --- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json +++ b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json @@ -44,7 +44,7 @@ ] }, { - "table": "edw.v_test_cal_dt", + "table": "edw.test_cal_dt", "columns": [ "cal_dt", "week_beg_dt" http://git-wip-us.apache.org/repos/asf/kylin/blob/576ec1b0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java index e605e5e..2cecfd8 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java @@ -287,21 +287,24 @@ public class CreateHTableJob extends AbstractHadoopJob { fs.mkdirs(outputFolder); } - float hfileSizeMB = kylinConfig.getHBaseHFileSizeGB() * 1024; - - if (kylinConfig.isDevEnv()) { - hfileSizeMB = hfileSizeMB/1024; + final float hfileSizeGB = kylinConfig.getHBaseHFileSizeGB(); + float hfileSizeMB = hfileSizeGB * 1024; + if (hfileSizeMB > mbPerRegion) { + hfileSizeMB = mbPerRegion; } + if (hfileSizeMB > 0.0 && kylinConfig.isDevEnv()) { + hfileSizeMB = mbPerRegion / 2; + } int compactionThreshold = Integer.valueOf(HBaseConnection.getCurrentHBaseConfiguration().get("hbase.hstore.compactionThreshold", "3")); - if (hfileSizeMB * compactionThreshold < mbPerRegion) { + logger.info("hbase.hstore.compactionThreshold is " + compactionThreshold); + if (hfileSizeMB > 0.0 && hfileSizeMB * compactionThreshold < mbPerRegion) { hfileSizeMB = mbPerRegion / compactionThreshold; } if (hfileSizeMB <= 0) { hfileSizeMB = mbPerRegion; } - logger.info("hfileSizeMB:" + hfileSizeMB); final Path hfilePartitionFile = new Path(outputFolder, "part-r-00000_hfile"); short regionCount = (short) innerRegionSplits.size();
