This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new fbe90c3 Revert "KYLIN-4097: Throw exception when too many dict slice
evictions in AppendTrieDictionary"
fbe90c3 is described below
commit fbe90c3121223e8923d7f8c90d3a3b70ab46695e
Author: Xiaoyuan Gu <xiaoyuan...@kyligence.io>
AuthorDate: Thu Oct 17 17:52:00 2019 +0800
Revert "KYLIN-4097: Throw exception when too many dict slice evictions in
AppendTrieDictionary"
This reverts commit aabf00ecbbe5dd3d0920f4b3b9cc875bf89116e4.
---
.../org/apache/kylin/common/KylinConfigBase.java | 9 -----
.../apache/kylin/dict/AppendTrieDictionary.java | 31 +---------------
.../kylin/dict/AppendTrieDictionaryTest.java | 41 ----------------------
3 files changed, 1 insertion(+), 80 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 1e61b19..6f73024 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -500,10 +500,6 @@ public abstract class KylinConfigBase implements
Serializable {
return Long.parseLong(getOptional("kylin.dictionary.max-cache-entry",
"3000"));
}
- public int getCachedDictMaxSize() {
- return Integer.parseInt(getOptional("kylin.dictionary.max-cache-size",
"-1"));
- }
-
public boolean isGrowingDictEnabled() {
return
Boolean.parseBoolean(this.getOptional("kylin.dictionary.growing-enabled",
FALSE));
}
@@ -556,11 +552,6 @@ public abstract class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(this.getOptional("kylin.dictionary.shrunken-from-global-enabled",
TRUE));
}
- public int getDictionarySliceEvicationThreshold() {
- return
Integer.parseInt(getOptional("kylin.dictionary.slice.eviction.threshold", "5"));
- }
-
-
//
============================================================================
// mr-hive dict
//
============================================================================
diff --git
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 8e89fd8..3a55961 100644
---
a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++
b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -39,7 +39,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheStats;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.cache.RemovalListener;
@@ -74,8 +73,6 @@ public class AppendTrieDictionary<T> extends
CacheDictionary<T> {
transient private GlobalDictMetadata metadata;
transient private LoadingCache<AppendDictSliceKey, AppendDictSlice>
dictCache;
- private int evictionThreshold = 0;
-
public void init(String baseDir) throws IOException {
this.baseDir = convertToAbsolutePath(baseDir);
final GlobalDictStore globalDictStore = new
GlobalDictHDFSStore(this.baseDir);
@@ -90,15 +87,7 @@ public class AppendTrieDictionary<T> extends
CacheDictionary<T> {
final Path latestVersionPath =
globalDictStore.getVersionDir(latestVersion);
this.metadata = globalDictStore.getMetadata(latestVersion);
this.bytesConvert = metadata.bytesConverter;
-
- // see: https://github.com/google/guava/wiki/CachesExplained
- CacheBuilder cacheBuilder =
CacheBuilder.newBuilder().softValues().recordStats();
- int cacheMaximumSize =
KylinConfig.getInstanceFromEnv().getCachedDictMaxSize();
- if (cacheMaximumSize > 0) {
- cacheBuilder = cacheBuilder.maximumSize(cacheMaximumSize);
- logger.info("Set dict cache maximum size to " + cacheMaximumSize);
- }
- this.dictCache = cacheBuilder
+ this.dictCache = CacheBuilder.newBuilder().softValues()
.removalListener(new RemovalListener<AppendDictSliceKey,
AppendDictSlice>() {
@Override
public void
onRemoval(RemovalNotification<AppendDictSliceKey, AppendDictSlice>
notification) {
@@ -115,7 +104,6 @@ public class AppendTrieDictionary<T> extends
CacheDictionary<T> {
return slice;
}
});
- this.evictionThreshold =
KylinConfig.getInstanceFromEnv().getDictionarySliceEvicationThreshold();
}
@Override
@@ -131,26 +119,9 @@ public class AppendTrieDictionary<T> extends
CacheDictionary<T> {
} catch (ExecutionException e) {
throw new IllegalStateException("Failed to load slice with key " +
sliceKey, e.getCause());
}
- CacheStats stats = dictCache.stats();
- if (evictionThreshold > 0 && stats.evictionCount() > evictionThreshold
* metadata.sliceFileMap.size()
- && stats.loadCount() > (evictionThreshold + 1) *
metadata.sliceFileMap.size()) {
- logger.warn(
- "Too many dict slice evictions and reloads, maybe the
memory is not enough to hold all the dictionary");
- throw new RuntimeException("Too many dict slice evictions: " +
stats + " for "
- + metadata.sliceFileMap.size() + " dict slices. "
- + "Maybe the memory is not enough to hold all the
dictionary, try to enlarge the mapreduce/spark executor memory.");
- }
return slice.getIdFromValueBytesImpl(value, offset, len, roundingFlag);
}
- public CacheStats getCacheStats() {
- return dictCache.stats();
- }
-
- public GlobalDictMetadata getDictMetadata() {
- return metadata;
- }
-
@Override
public int getMinId() {
return metadata.baseId;
diff --git
a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index 7907da8..7e5421a 100644
---
a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++
b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -592,45 +592,4 @@ public class AppendTrieDictionaryTest extends
LocalFileMetadataTestCase {
}
}
- @Test
- public void testTooManySliceEvictions() throws IOException {
-
KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.max-cache-size",
"3");
- AppendTrieDictionaryBuilder builder = createBuilder();
- for (int i = 0 ; i < 100000; i++) {
- builder.addValue(Integer.toString(i));
- }
- AppendTrieDictionary dict = builder.build(0);
-
- assertEquals(4, dict.getDictMetadata().sliceFileMap.size());
- assertEquals(1, dict.getIdFromValue("0", 0));
- assertEquals(0, dict.getCacheStats().evictionCount());
- assertEquals(1, dict.getCacheStats().loadCount());
-
-
- List<String> keys = new ArrayList<>(100000);
- for (int i = 0 ; i < 100000; i++) {
- keys.add(Integer.toString(i));
- }
- Collections.sort(keys);
- for (String key : keys) {
- assertEquals(Integer.parseInt(key) + 1, dict.getIdFromValue(key,
0));
- }
- assertEquals(1, dict.getCacheStats().evictionCount());
- assertEquals(4, dict.getCacheStats().loadCount());
-
- // out of order
- Collections.shuffle(keys);
- try {
- for (String key : keys) {
- assertEquals(Integer.parseInt(key) + 1,
dict.getIdFromValue(key, 0));
- }
- assertFalse("Should throw RuntimeException for too many dict slice
evictions", true);
- } catch (RuntimeException e) {
- assertEquals("Too many dict slice evictions",
e.getMessage().substring(0, 29));
- }
- assertEquals(22, dict.getCacheStats().evictionCount());
- assertEquals(25, dict.getCacheStats().loadCount());
-
-
KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.max-cache-size",
"-1");
- }
}
