[3/3] kylin git commit: KYLIN-1591 enabling different block size setting for small column families

[mahongbin]

KYLIN-1591 enabling different block size setting for small column families


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3aad1fc6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3aad1fc6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3aad1fc6

Branch: refs/heads/yang-m1
Commit: 3aad1fc60cc0278833d4b5e0e5f2db02788cd3ba
Parents: ad09341
Author: Hongbin Ma 
Authored: Mon Apr 18 15:40:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 18 15:51:31 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java  | 10 +++---
 .../apache/kylin/cube/model/HBaseColumnFamilyDesc.java | 13 +
 .../java/org/apache/kylin/gridtable/GTScanRequest.java |  3 ++-
 .../kylin/storage/hbase/steps/CubeHTableUtil.java  |  9 ++---
 4 files changed, 28 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 94d8f62..23f0fa7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -481,7 +481,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getQueryScanFuzzyKeyMax() {
-return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200"));
+return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200"));
 }
 
 public int getHBaseKeyValueSize() {
@@ -546,8 +546,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.hbase.default.encoding", "FAST_DIFF");
 }
 
-public String getHbaseDefaultBlockSize() {
-return getOptional("kylin.hbase.default.block.size", "4194304");
+public int getHbaseDefaultBlockSize() {
+return Integer.valueOf(getOptional("kylin.hbase.default.block.size", 
"1048576"));
+}
+
+public int getHbaseSmallFamilyBlockSize() {
+return 
Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536"));
 }
 
 public boolean isHiveKeepFlatTable() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
index 5db2a9b..c5b2e19 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
@@ -20,6 +20,8 @@ package org.apache.kylin.cube.model;
 
 import java.util.Arrays;
 
+import org.apache.kylin.metadata.model.MeasureDesc;
+
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -55,4 +57,15 @@ public class HBaseColumnFamilyDesc {
 return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + 
Arrays.toString(columns) + "]";
 }
 
+public boolean isMemoryHungry() {
+for (HBaseColumnDesc hBaseColumnDesc : columns) {
+for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) {
+if 
(measureDesc.getFunction().getMeasureType().isMemoryHungry()) {
+return true;
+}
+}
+}
+return false;
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/3aad1fc6/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 97900c1..befa866 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -153,7 +153,8 @@ public class GTScanRequest {
 }
 
 if (!doAggr) {//Skip reading this section if you're not profiling! 
-int scanned = lookAndForget(result);
+int scanned = result.getScannedRowCount();
+lookAndForget(result);
 return new EmptyGTScanner(scanned);
 }
 

kylin git commit: KYLIN-1591 enabling different block size setting for small column families

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 41014d6a5 -> 22e6969cb


KYLIN-1591 enabling different block size setting for small column families


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/22e6969c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/22e6969c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/22e6969c

Branch: refs/heads/master
Commit: 22e6969cb30f2146b23a9c81701b8a39dba15fd4
Parents: 41014d6
Author: Hongbin Ma 
Authored: Mon Apr 18 15:40:05 2016 +0800
Committer: Hongbin Ma 
Committed: Mon Apr 18 15:40:54 2016 +0800

--
 .../java/org/apache/kylin/common/KylinConfigBase.java  | 10 +++---
 .../apache/kylin/cube/model/HBaseColumnFamilyDesc.java | 13 +
 .../java/org/apache/kylin/gridtable/GTScanRequest.java |  3 ++-
 .../kylin/storage/hbase/steps/CubeHTableUtil.java  |  9 ++---
 4 files changed, 28 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 51aa8aa..4c7995b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -489,7 +489,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 }
 
 public int getQueryScanFuzzyKeyMax() {
-return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200"));
+return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200"));
 }
 
 public int getQueryStorageVisitScanRangeMax() {
@@ -558,8 +558,12 @@ abstract public class KylinConfigBase implements 
Serializable {
 return getOptional("kylin.hbase.default.encoding", "FAST_DIFF");
 }
 
-public String getHbaseDefaultBlockSize() {
-return getOptional("kylin.hbase.default.block.size", "4194304");
+public int getHbaseDefaultBlockSize() {
+return Integer.valueOf(getOptional("kylin.hbase.default.block.size", 
"1048576"));
+}
+
+public int getHbaseSmallFamilyBlockSize() {
+return 
Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536"));
 }
 
 public boolean isHiveKeepFlatTable() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
 
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
index 5db2a9b..c5b2e19 100644
--- 
a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
+++ 
b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
@@ -20,6 +20,8 @@ package org.apache.kylin.cube.model;
 
 import java.util.Arrays;
 
+import org.apache.kylin.metadata.model.MeasureDesc;
+
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -55,4 +57,15 @@ public class HBaseColumnFamilyDesc {
 return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + 
Arrays.toString(columns) + "]";
 }
 
+public boolean isMemoryHungry() {
+for (HBaseColumnDesc hBaseColumnDesc : columns) {
+for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) {
+if 
(measureDesc.getFunction().getMeasureType().isMemoryHungry()) {
+return true;
+}
+}
+}
+return false;
+}
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/22e6969c/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 5681057..bc478c9 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -161,7 +161,8 @@ public class GTScanRequest {
 }
 
 if (!doAggr) {//Skip reading this section if you're not profiling! 
-int scanned = lookAndForget(result);
+int scanned = result.getScannedRowCount();
+