KYLIN-1843: support LRU cache on SnapshotManager Signed-off-by: Li Yang <liy...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6b45c945 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6b45c945 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6b45c945 Branch: refs/heads/master Commit: 6b45c945af9948c17a971d530c0153b3564f1985 Parents: 7dc8b06 Author: Yiming Liu <liuyiming....@gmail.com> Authored: Thu Jul 7 12:30:25 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Fri Jul 8 18:13:55 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 4 ++ .../kylin/dict/lookup/SnapshotManager.java | 42 +++++++++++++++----- 2 files changed, 37 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/6b45c945/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index bfbaaa4..6b528f7 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -502,6 +502,10 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", "3000")); } + public int getCachedSnapshotMaxEntrySize(){ + return Integer.parseInt(getOptional("kylin.snapshot.cache.max.entry", "500")); + } + public boolean getQueryRunLocalCoprocessor() { return Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/6b45c945/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java index 6e93ae4..c7b0d26 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java @@ -21,6 +21,8 @@ package org.apache.kylin.dict.lookup; import java.io.IOException; import java.util.NavigableSet; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; @@ -31,6 +33,12 @@ import org.apache.kylin.source.ReadableTable.TableSignature; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; + /** * @author yangli9 */ @@ -61,33 +69,49 @@ public class SnapshotManager { // ============================================================================ private KylinConfig config; - private ConcurrentHashMap<String, SnapshotTable> snapshotCache; // resource + private LoadingCache<String, SnapshotTable> snapshotCache; // resource // path ==> // SnapshotTable private SnapshotManager(KylinConfig config) { this.config = config; - snapshotCache = new ConcurrentHashMap<String, SnapshotTable>(); + this.snapshotCache = CacheBuilder.newBuilder().removalListener(new RemovalListener<String, SnapshotTable>() { + @Override + public void onRemoval(RemovalNotification<String, SnapshotTable> notification) { + SnapshotManager.logger.info("Snapshot with resource path " + notification.getKey() + " is removed due to " + notification.getCause()); + } + }).maximumSize(config.getCachedSnapshotMaxEntrySize())// + .expireAfterWrite(1, TimeUnit.DAYS).build(new CacheLoader<String, SnapshotTable>() { + @Override + public SnapshotTable load(String key) throws Exception { + SnapshotTable snapshotTable = SnapshotManager.this.load(key, true); + return snapshotTable; + } + }); } public void wipeoutCache() { - snapshotCache.clear(); + snapshotCache.invalidateAll(); } public SnapshotTable getSnapshotTable(String resourcePath) throws IOException { - SnapshotTable r = snapshotCache.get(resourcePath); - if (r == null) { - r = load(resourcePath, true); - snapshotCache.put(resourcePath, r); + try { + SnapshotTable r = snapshotCache.get(resourcePath); + if (r == null) { + r = load(resourcePath, true); + snapshotCache.put(resourcePath, r); + } + return r; + } catch (ExecutionException e) { + throw new RuntimeException(e.getCause()); } - return r; } public void removeSnapshot(String resourcePath) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); store.deleteResource(resourcePath); - snapshotCache.remove(resourcePath); + snapshotCache.invalidate(resourcePath); } public SnapshotTable buildSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException {
