kylin git commit: KYLIN-1297 correct LoadingCache settings

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 2f5d89a8e -> eea691651


KYLIN-1297 correct LoadingCache settings


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eea69165
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eea69165
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eea69165

Branch: refs/heads/1.x-staging
Commit: eea6916518a59ce137ac6d26bd539e39bb871a6c
Parents: 2f5d89a
Author: honma 
Authored: Thu Jan 21 10:28:54 2016 +0800
Committer: honma 
Committed: Thu Jan 21 10:29:05 2016 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  5 
 .../apache/kylin/dict/DictionaryManager.java| 31 +++-
 .../storage/hbase/CubeSegmentTupleIterator.java |  1 -
 3 files changed, 22 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eea69165/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index e0f774a..790e7a5 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -466,6 +466,11 @@ public class KylinConfig {
 return this.getOptional("kylin.server.mode", "all");
 }
 
+public int getCachedDictMaxEntrySize() {
+return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
+}
+
+
 public int getDictionaryMaxCardinality() {
 return 
Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "500"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eea69165/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index aa0bc5d..9294e03 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -28,9 +28,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
 import org.apache.commons.compress.utils.IOUtils;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -49,6 +46,10 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
 public class DictionaryManager {
 
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryManager.class);
@@ -84,17 +85,19 @@ public class DictionaryManager {
 
 private DictionaryManager(KylinConfig config) {
 this.config = config;
-this.dictCache = 
CacheBuilder.newBuilder().weakValues().expireAfterWrite(10, 
TimeUnit.MINUTES).build(new CacheLoader() {
-@Override
-public DictionaryInfo load(String key) throws Exception {
-DictionaryInfo dictInfo = DictionaryManager.this.load(key, 
true);
-if (dictInfo == null) {
-return NONE_INDICATOR;
-} else {
-return dictInfo;
-}
-}
-});
+this.dictCache = 
CacheBuilder.newBuilder().maximumSize(KylinConfig.getInstanceFromEnv().getCachedDictMaxEntrySize())//
+.expireAfterWrite(1, TimeUnit.DAYS).build(new 
CacheLoader() {
+
+@Override
+public DictionaryInfo load(String key) throws Exception {
+DictionaryInfo dictInfo = 
DictionaryManager.this.load(key, true);
+if (dictInfo == null) {
+return NONE_INDICATOR;
+} else {
+return dictInfo;
+}
+}
+});
 }
 
 public Dictionary getDictionary(String resourcePath) throws IOException 
{

http://git-wip-us.apache.org/repos/asf/kylin/blob/eea69165/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
--
diff --git 

[08/32] kylin git commit: KYLIN-1297 correct LoadingCache settings

[wangxiaoyu]

KYLIN-1297 correct LoadingCache settings


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ce734019
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ce734019
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ce734019

Branch: refs/heads/KYLIN-1122
Commit: ce734019e985d04a47178f12aeb8e8622fdd3eee
Parents: cb0c36a
Author: honma 
Authored: Fri Jan 8 15:48:58 2016 +0800
Committer: Xiaoyu Wang 
Committed: Mon Jan 18 13:38:30 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java  | 5 +
 .../src/main/java/org/apache/kylin/dict/DictionaryManager.java  | 5 +++--
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/ce734019/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 96c76f0..f6775e8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -408,6 +408,11 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }
 
+public int getCachedDictMaxEntrySize() {
+return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
+}
+
+
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/ce734019/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index cb58721..cca8341 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -83,7 +83,8 @@ public class DictionaryManager {
 
 private DictionaryManager(KylinConfig config) {
 this.config = config;
-this.dictCache = 
CacheBuilder.newBuilder().weakValues().expireAfterWrite(10, 
TimeUnit.MINUTES).build(new CacheLoader() {
+this.dictCache = 
CacheBuilder.newBuilder().maximumSize(KylinConfig.getInstanceFromEnv().getCachedDictMaxEntrySize())//
+.expireAfterWrite(1, TimeUnit.DAYS).build(new 
CacheLoader() {
 @Override
 public DictionaryInfo load(String key) throws Exception {
 DictionaryInfo dictInfo = DictionaryManager.this.load(key, 
true);
@@ -357,7 +358,7 @@ public class DictionaryManager {
 DictionaryInfo load(String resourcePath, boolean loadDictObj) throws 
IOException {
 ResourceStore store = MetadataManager.getInstance(config).getStore();
 
-//logger.debug("Going to load DictionaryInfo from " + resourcePath);
+logger.debug("Loading DictionaryInfo(loadDictObj:" + loadDictObj + ") 
from " + resourcePath);
 DictionaryInfo info = store.getResource(resourcePath, 
DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : 
DictionaryInfoSerializer.INFO_SERIALIZER);
 
 //if (loadDictObj)

[1/9] kylin git commit: KYLIN-1297 correct LoadingCache settings

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 05b7ca560 -> bc8ea7817


KYLIN-1297 correct LoadingCache settings


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2d3d8597
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2d3d8597
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2d3d8597

Branch: refs/heads/2.x-staging
Commit: 2d3d859724f0f8fde2113ef187e14635e52dfd00
Parents: 05b7ca5
Author: honma 
Authored: Fri Jan 8 15:48:58 2016 +0800
Committer: honma 
Committed: Wed Jan 13 17:48:31 2016 +0800

--
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java  | 5 +
 .../src/main/java/org/apache/kylin/dict/DictionaryManager.java  | 5 +++--
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2d3d8597/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 96c76f0..f6775e8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -408,6 +408,11 @@ public class KylinConfigBase implements Serializable {
 return 
Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10"));
 }
 
+public int getCachedDictMaxEntrySize() {
+return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
+}
+
+
 public boolean getQueryRunLocalCoprocessor() {
 return 
Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2d3d8597/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index cb58721..cca8341 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -83,7 +83,8 @@ public class DictionaryManager {
 
 private DictionaryManager(KylinConfig config) {
 this.config = config;
-this.dictCache = 
CacheBuilder.newBuilder().weakValues().expireAfterWrite(10, 
TimeUnit.MINUTES).build(new CacheLoader() {
+this.dictCache = 
CacheBuilder.newBuilder().maximumSize(KylinConfig.getInstanceFromEnv().getCachedDictMaxEntrySize())//
+.expireAfterWrite(1, TimeUnit.DAYS).build(new 
CacheLoader() {
 @Override
 public DictionaryInfo load(String key) throws Exception {
 DictionaryInfo dictInfo = DictionaryManager.this.load(key, 
true);
@@ -357,7 +358,7 @@ public class DictionaryManager {
 DictionaryInfo load(String resourcePath, boolean loadDictObj) throws 
IOException {
 ResourceStore store = MetadataManager.getInstance(config).getStore();
 
-//logger.debug("Going to load DictionaryInfo from " + resourcePath);
+logger.debug("Loading DictionaryInfo(loadDictObj:" + loadDictObj + ") 
from " + resourcePath);
 DictionaryInfo info = store.getResource(resourcePath, 
DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : 
DictionaryInfoSerializer.INFO_SERIALIZER);
 
 //if (loadDictObj)