subject:"kylin git commit\: KYLIN\-1307 revisit growing dictionary"

[13/32] kylin git commit: KYLIN-1307 revisit growing dictionary

2016-01-17 Thread wangxiaoyu

KYLIN-1307 revisit growing dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/06f79e6f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/06f79e6f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/06f79e6f

Branch: refs/heads/KYLIN-1122
Commit: 06f79e6fc35f35d6446fb4c2f1f463192cf3d124
Parents: c02582f
Author: honma 
Authored: Tue Jan 12 16:06:39 2016 +0800
Committer: Xiaoyu Wang 
Committed: Mon Jan 18 13:38:30 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 +-
 .../apache/kylin/dict/DictionaryManager.java| 58 +++-
 .../apache/kylin/rest/service/CacheService.java |  2 +
 3 files changed, 51 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/06f79e6f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f6775e8..bfad306 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -412,7 +412,6 @@ public class KylinConfigBase implements Serializable {
 return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
 }
 
-
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }
@@ -453,6 +452,10 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
 }
 
+public boolean isGrowingDictEnabled() {
+return 
Boolean.parseBoolean(this.getOptional("kylin.dict.growing.enabled", "false"));
+}
+
 public int getHBaseScanMaxResultSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/06f79e6f/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 630b2b2..d49e43d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -140,26 +140,58 @@ public class DictionaryManager {
 
 initDictInfo(newDict, newDictInfo);
 
-DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
-if (largestDictInfo != null) {
-largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
-Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
-if (largestDictObject.contains(newDict)) {
-logger.info("dictionary content " + newDict + ", is contained 
by  dictionary at " + largestDictInfo.getResourcePath());
-return largestDictInfo;
-} else if (newDict.contains(largestDictObject)) {
-logger.info("dictionary content " + newDict + " is by far the 
largest, save it");
-return saveNewDict(newDictInfo);
+if (KylinConfig.getInstanceFromEnv().isGrowingDictEnabled()) {
+DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
+if (largestDictInfo != null) {
+largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
+Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
+if (largestDictObject.contains(newDict)) {
+logger.info("dictionary content " + newDict + ", is 
contained by  dictionary at " + largestDictInfo.getResourcePath());
+return largestDictInfo;
+} else if (newDict.contains(largestDictObject)) {
+logger.info("dictionary content " + newDict + " is by far 
the largest, save it");
+return saveNewDict(newDictInfo);
+} else {
+logger.info("merge dict and save...");
+return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+}
 } else {
-logger.info("merge dict and save...");
-return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+logger.info("first dict of this column, save

[6/9] kylin git commit: KYLIN-1307 revisit growing dictionary

2016-01-13 Thread mahongbin

KYLIN-1307 revisit growing dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c5784b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c5784b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c5784b2

Branch: refs/heads/2.x-staging
Commit: 1c5784b28ff9bbeb4a1ee9d4db3b05461641165d
Parents: 142eaf4
Author: honma 
Authored: Tue Jan 12 16:06:39 2016 +0800
Committer: honma 
Committed: Wed Jan 13 17:52:57 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 +-
 .../apache/kylin/dict/DictionaryManager.java| 58 +++-
 .../apache/kylin/rest/service/CacheService.java |  2 +
 3 files changed, 51 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1c5784b2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f6775e8..bfad306 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -412,7 +412,6 @@ public class KylinConfigBase implements Serializable {
 return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
 }
 
-
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }
@@ -453,6 +452,10 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
 }
 
+public boolean isGrowingDictEnabled() {
+return 
Boolean.parseBoolean(this.getOptional("kylin.dict.growing.enabled", "false"));
+}
+
 public int getHBaseScanMaxResultSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/1c5784b2/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 630b2b2..d49e43d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -140,26 +140,58 @@ public class DictionaryManager {
 
 initDictInfo(newDict, newDictInfo);
 
-DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
-if (largestDictInfo != null) {
-largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
-Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
-if (largestDictObject.contains(newDict)) {
-logger.info("dictionary content " + newDict + ", is contained 
by  dictionary at " + largestDictInfo.getResourcePath());
-return largestDictInfo;
-} else if (newDict.contains(largestDictObject)) {
-logger.info("dictionary content " + newDict + " is by far the 
largest, save it");
-return saveNewDict(newDictInfo);
+if (KylinConfig.getInstanceFromEnv().isGrowingDictEnabled()) {
+DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
+if (largestDictInfo != null) {
+largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
+Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
+if (largestDictObject.contains(newDict)) {
+logger.info("dictionary content " + newDict + ", is 
contained by  dictionary at " + largestDictInfo.getResourcePath());
+return largestDictInfo;
+} else if (newDict.contains(largestDictObject)) {
+logger.info("dictionary content " + newDict + " is by far 
the largest, save it");
+return saveNewDict(newDictInfo);
+} else {
+logger.info("merge dict and save...");
+return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+}
 } else {
-logger.info("merge dict and save...");
-return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+logger.info("first dict of this column, save it

kylin git commit: KYLIN-1307 revisit growing dictionary

2016-01-12 Thread mahongbin

Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 3904a1767 -> 9f743fb78


KYLIN-1307 revisit growing dictionary


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9f743fb7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9f743fb7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9f743fb7

Branch: refs/heads/2.0-rc
Commit: 9f743fb7871d02f2c06e8fe21d288de1f502c939
Parents: 3904a17
Author: honma 
Authored: Tue Jan 12 16:06:39 2016 +0800
Committer: honma 
Committed: Tue Jan 12 16:06:39 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 +-
 .../apache/kylin/dict/DictionaryManager.java| 58 +++-
 .../apache/kylin/rest/service/CacheService.java |  2 +
 3 files changed, 51 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/9f743fb7/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a5ca690..4c640f3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -412,7 +412,6 @@ public class KylinConfigBase implements Serializable {
 return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
 }
 
-
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }
@@ -453,6 +452,10 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
 }
 
+public boolean isGrowingDictEnabled() {
+return 
Boolean.parseBoolean(this.getOptional("kylin.dict.growing.enabled", "false"));
+}
+
 public int getHBaseScanMaxResultSize() {
 return 
Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 
1024 * 1024))); // 5 MB
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9f743fb7/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index b3cd634..cd12f01 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -139,26 +139,58 @@ public class DictionaryManager {
 
 initDictInfo(newDict, newDictInfo);
 
-DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
-if (largestDictInfo != null) {
-largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
-Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
-if (largestDictObject.contains(newDict)) {
-logger.info("dictionary content " + newDict + ", is contained 
by  dictionary at " + largestDictInfo.getResourcePath());
-return largestDictInfo;
-} else if (newDict.contains(largestDictObject)) {
-logger.info("dictionary content " + newDict + " is by far the 
largest, save it");
-return saveNewDict(newDictInfo);
+if (KylinConfig.getInstanceFromEnv().isGrowingDictEnabled()) {
+DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
+if (largestDictInfo != null) {
+largestDictInfo = 
getDictionaryInfo(largestDictInfo.getResourcePath());
+Dictionary largestDictObject = 
largestDictInfo.getDictionaryObject();
+if (largestDictObject.contains(newDict)) {
+logger.info("dictionary content " + newDict + ", is 
contained by  dictionary at " + largestDictInfo.getResourcePath());
+return largestDictInfo;
+} else if (newDict.contains(largestDictObject)) {
+logger.info("dictionary content " + newDict + " is by far 
the largest, save it");
+return saveNewDict(newDictInfo);
+} else {
+logger.info("merge dict and save...");
+return mergeDictionary(Lists.newArrayList(newDictInfo, 
largestDictInfo));
+}
 } else {
-logger.info("merge dict and save...");
-return mergeDictionary(Lists.newArrayList(newDictInfo,

[13/32] kylin git commit: KYLIN-1307 revisit growing dictionary

[6/9] kylin git commit: KYLIN-1307 revisit growing dictionary

kylin git commit: KYLIN-1307 revisit growing dictionary

3 matches

Site Navigation

Mail list logo

Footer information