minor, fix CI occasionally fail issue
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f5074ee Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f5074ee Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f5074ee Branch: refs/heads/2.x-staging Commit: 3f5074ee1568d5b0ba50d70d5c35319cd8223cc9 Parents: ab9d579 Author: honma <ho...@ebay.com> Authored: Thu Feb 25 14:13:48 2016 +0800 Committer: honma <ho...@ebay.com> Committed: Fri Feb 26 17:54:37 2016 +0800 ---------------------------------------------------------------------- .../dict/TupleFilterDictionaryTranslater.java | 165 ------------------ .../dict/TupleFilterFunctionTranslator.java | 166 +++++++++++++++++++ .../metadata/filter/FunctionTupleFilter.java | 5 +- .../metadata/filter/function/BuiltInMethod.java | 33 ++-- .../cache/AbstractCacheFledgedQuery.java | 32 +--- .../kylin/storage/cache/DynamicCacheTest.java | 15 +- .../kylin/storage/cache/StaticCacheTest.java | 19 ++- .../kylin/storage/hbase/ITStorageTest.java | 11 +- .../common/coprocessor/FilterDecorator.java | 4 +- .../hbase/cube/v2/CubeSegmentScanner.java | 4 +- 10 files changed, 226 insertions(+), 228 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java deleted file mode 100644 index 9ef360d..0000000 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterDictionaryTranslater.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.dict; - -import com.google.common.primitives.Primitives; -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.FunctionTupleFilter; -import org.apache.kylin.metadata.filter.ITupleFilterTranslator; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ListIterator; - -/** - * Created by dongli on 1/7/16. - */ -public class TupleFilterDictionaryTranslater implements ITupleFilterTranslator { - public static final Logger logger = LoggerFactory.getLogger(TupleFilterDictionaryTranslater.class); - - private IDictionaryAware dictionaryAware; - - public TupleFilterDictionaryTranslater(IDictionaryAware dictionaryAware) { - this.dictionaryAware = dictionaryAware; - } - - @Override - public TupleFilter translate(TupleFilter tupleFilter) { - TupleFilter translated = null; - if (tupleFilter instanceof CompareTupleFilter) { - translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter); - if (translated != null) { - logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); - } - } else if (tupleFilter instanceof FunctionTupleFilter) { - translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter); - if (translated != null) { - logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); - } - } else if (tupleFilter instanceof LogicalTupleFilter) { - ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator(); - while (childIterator.hasNext()) { - TupleFilter tempTranslated = translate(childIterator.next()); - if (tempTranslated != null) - childIterator.set(tempTranslated); - } - } - return translated == null ? tupleFilter : translated; - } - - private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) { - if (!functionTupleFilter.isValid()) - return null; - - TblColRef columnRef = functionTupleFilter.getColumn(); - Dictionary<?> dict = dictionaryAware.getDictionary(columnRef); - if (dict == null) - return null; - - CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); - translated.addChild(new ColumnTupleFilter(columnRef)); - - try { - for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { - Object dictVal = dict.getValueFromId(i); - if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) { - translated.addChild(new ConstantTupleFilter(dictVal)); - } - } - } catch (Exception e) { - logger.debug(e.getMessage()); - return null; - } - return translated; - } - - @SuppressWarnings("unchecked") - private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) { - if (compTupleFilter.getFunction() == null) - return null; - - FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction(); - if (!functionTupleFilter.isValid()) - return null; - - TblColRef columnRef = functionTupleFilter.getColumn(); - Dictionary<?> dict = dictionaryAware.getDictionary(columnRef); - if (dict == null) - return null; - - CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); - translated.addChild(new ColumnTupleFilter(columnRef)); - - try { - for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { - Object dictVal = dict.getValueFromId(i); - Object computedVal = functionTupleFilter.invokeFunction(dictVal); - Class clazz = Primitives.wrap(computedVal.getClass()); - Object targetVal = compTupleFilter.getFirstValue(); - if (Primitives.isWrapperType(clazz)) - targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue())); - - int comp = ((Comparable) computedVal).compareTo(targetVal); - boolean compResult = false; - switch (compTupleFilter.getOperator()) { - case EQ: - compResult = comp == 0; - break; - case NEQ: - compResult = comp != 0; - break; - case LT: - compResult = comp < 0; - break; - case LTE: - compResult = comp <= 0; - break; - case GT: - compResult = comp > 0; - break; - case GTE: - compResult = comp >= 0; - break; - case IN: - compResult = compTupleFilter.getValues().contains(computedVal.toString()); - break; - case NOTIN: - compResult = !compTupleFilter.getValues().contains(computedVal.toString()); - break; - default: - break; - } - if (compResult) { - translated.addChild(new ConstantTupleFilter(dictVal)); - } - } - } catch (Exception e) { - logger.debug(e.getMessage()); - return null; - } - return translated; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java new file mode 100644 index 0000000..1c96dd4 --- /dev/null +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TupleFilterFunctionTranslator.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dict; + +import java.util.ListIterator; + +import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.metadata.filter.ColumnTupleFilter; +import org.apache.kylin.metadata.filter.CompareTupleFilter; +import org.apache.kylin.metadata.filter.ConstantTupleFilter; +import org.apache.kylin.metadata.filter.FunctionTupleFilter; +import org.apache.kylin.metadata.filter.ITupleFilterTranslator; +import org.apache.kylin.metadata.filter.LogicalTupleFilter; +import org.apache.kylin.metadata.filter.TupleFilter; +import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; +import org.apache.kylin.metadata.model.TblColRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.primitives.Primitives; + +/** + * only take effect when the compare filter has function + */ +public class TupleFilterFunctionTranslator implements ITupleFilterTranslator { + public static final Logger logger = LoggerFactory.getLogger(TupleFilterFunctionTranslator.class); + + private IDictionaryAware dictionaryAware; + + public TupleFilterFunctionTranslator(IDictionaryAware dictionaryAware) { + this.dictionaryAware = dictionaryAware; + } + + @Override + public TupleFilter translate(TupleFilter tupleFilter) { + TupleFilter translated = null; + if (tupleFilter instanceof CompareTupleFilter) { + translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter); + if (translated != null) { + logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); + } + } else if (tupleFilter instanceof FunctionTupleFilter) { + translated = translateFunctionTupleFilter((FunctionTupleFilter) tupleFilter); + if (translated != null) { + logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}"); + } + } else if (tupleFilter instanceof LogicalTupleFilter) { + ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator(); + while (childIterator.hasNext()) { + TupleFilter tempTranslated = translate(childIterator.next()); + if (tempTranslated != null) + childIterator.set(tempTranslated); + } + } + return translated == null ? tupleFilter : translated; + } + + private TupleFilter translateFunctionTupleFilter(FunctionTupleFilter functionTupleFilter) { + if (!functionTupleFilter.isValid()) + return null; + + TblColRef columnRef = functionTupleFilter.getColumn(); + Dictionary<?> dict = dictionaryAware.getDictionary(columnRef); + if (dict == null) + return null; + + CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); + translated.addChild(new ColumnTupleFilter(columnRef)); + + try { + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { + Object dictVal = dict.getValueFromId(i); + if ((Boolean) functionTupleFilter.invokeFunction(dictVal)) { + translated.addChild(new ConstantTupleFilter(dictVal)); + } + } + } catch (Exception e) { + logger.debug(e.getMessage()); + return null; + } + return translated; + } + + @SuppressWarnings("unchecked") + private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) { + if (compTupleFilter.getFunction() == null) + return null; + + FunctionTupleFilter functionTupleFilter = compTupleFilter.getFunction(); + if (!functionTupleFilter.isValid()) + return null; + + TblColRef columnRef = functionTupleFilter.getColumn(); + Dictionary<?> dict = dictionaryAware.getDictionary(columnRef); + if (dict == null) + return null; + + CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN); + translated.addChild(new ColumnTupleFilter(columnRef)); + + try { + for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { + Object dictVal = dict.getValueFromId(i); + Object computedVal = functionTupleFilter.invokeFunction(dictVal); + Class clazz = Primitives.wrap(computedVal.getClass()); + Object targetVal = compTupleFilter.getFirstValue(); + if (Primitives.isWrapperType(clazz)) + targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue())); + + int comp = ((Comparable) computedVal).compareTo(targetVal); + boolean compResult = false; + switch (compTupleFilter.getOperator()) { + case EQ: + compResult = comp == 0; + break; + case NEQ: + compResult = comp != 0; + break; + case LT: + compResult = comp < 0; + break; + case LTE: + compResult = comp <= 0; + break; + case GT: + compResult = comp > 0; + break; + case GTE: + compResult = comp >= 0; + break; + case IN: + compResult = compTupleFilter.getValues().contains(computedVal.toString()); + break; + case NOTIN: + compResult = !compTupleFilter.getValues().contains(computedVal.toString()); + break; + default: + break; + } + if (compResult) { + translated.addChild(new ConstantTupleFilter(dictVal)); + } + } + } catch (Exception e) { + logger.debug(e.getMessage()); + return null; + } + return translated; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java index 15fcb72..30bef97 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FunctionTupleFilter.java @@ -35,9 +35,6 @@ import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; import com.google.common.primitives.Primitives; -/** - * Created by dongli on 11/11/15. - */ public class FunctionTupleFilter extends TupleFilter { public static final Logger logger = LoggerFactory.getLogger(FunctionTupleFilter.class); @@ -79,7 +76,7 @@ public class FunctionTupleFilter extends TupleFilter { if (columnContainerFilter instanceof ColumnTupleFilter) methodParams.set(colPosition, (Serializable) input); else if (columnContainerFilter instanceof FunctionTupleFilter) - methodParams.set(colPosition, (Serializable) ((FunctionTupleFilter) columnContainerFilter).invokeFunction((Serializable) input)); + methodParams.set(colPosition, (Serializable) ((FunctionTupleFilter) columnContainerFilter).invokeFunction(input)); return method.invoke(null, (Object[]) (methodParams.toArray())); } http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java index b927d8d..7b241cc 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java @@ -18,29 +18,21 @@ package org.apache.kylin.metadata.filter.function; -import com.google.common.collect.ImmutableMap; -import org.apache.commons.lang3.reflect.MethodUtils; - import java.lang.reflect.Method; import java.util.regex.Pattern; -/** - * Created by dongli on 11/13/15. - */ +import org.apache.commons.lang3.reflect.MethodUtils; + +import com.google.common.collect.ImmutableMap; + public enum BuiltInMethod { - UPPER(BuiltInMethod.class, "upper", String.class), - LOWER(BuiltInMethod.class, "lower", String.class), - SUBSTRING(BuiltInMethod.class, "substring", String.class, int.class, int.class), - CHAR_LENGTH(BuiltInMethod.class, "charLength", String.class), - LIKE(BuiltInMethod.class, "like", String.class, String.class), - INITCAP(BuiltInMethod.class, "initcap", String.class); + UPPER(BuiltInMethod.class, "upper", String.class), LOWER(BuiltInMethod.class, "lower", String.class), SUBSTRING(BuiltInMethod.class, "substring", String.class, int.class, int.class), CHAR_LENGTH(BuiltInMethod.class, "charLength", String.class), LIKE(BuiltInMethod.class, "like", String.class, String.class), INITCAP(BuiltInMethod.class, "initcap", String.class); public final Method method; public static final ImmutableMap<String, BuiltInMethod> MAP; static { - final ImmutableMap.Builder<String, BuiltInMethod> builder = - ImmutableMap.builder(); + final ImmutableMap.Builder<String, BuiltInMethod> builder = ImmutableMap.builder(); for (BuiltInMethod value : BuiltInMethod.values()) { if (value.method != null) { builder.put(value.name(), value); @@ -70,22 +62,22 @@ public enum BuiltInMethod { for (int i = 0; i < len; i++) { char curCh = s.charAt(i); final int c = (int) curCh; - if (start) { // curCh is whitespace or first character of word. + if (start) { // curCh is whitespace or first character of word. if (c > 47 && c < 58) { // 0-9 start = false; - } else if (c > 64 && c < 91) { // A-Z + } else if (c > 64 && c < 91) { // A-Z start = false; - } else if (c > 96 && c < 123) { // a-z + } else if (c > 96 && c < 123) { // a-z start = false; curCh = (char) (c - 32); // Uppercase this character } // else {} whitespace - } else { // Inside of a word or white space after end of word. + } else { // Inside of a word or white space after end of word. if (c > 47 && c < 58) { // 0-9 // noop - } else if (c > 64 && c < 91) { // A-Z + } else if (c > 64 && c < 91) { // A-Z curCh = (char) (c + 32); // Lowercase this character - } else if (c > 96 && c < 123) { // a-z + } else if (c > 96 && c < 123) { // a-z // noop } else { // whitespace start = true; @@ -116,5 +108,4 @@ public enum BuiltInMethod { return s.toLowerCase(); } - } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java index 6ba76c4..a5ca800 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java @@ -5,9 +5,6 @@ import net.sf.ehcache.CacheManager; import net.sf.ehcache.Element; import net.sf.ehcache.Status; import net.sf.ehcache.config.CacheConfiguration; -import net.sf.ehcache.config.Configuration; -import net.sf.ehcache.config.PersistenceConfiguration; -import net.sf.ehcache.store.MemoryStoreEvictionPolicy; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.realization.StreamSQLDigest; @@ -21,6 +18,7 @@ import org.slf4j.LoggerFactory; */ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTupleItrListener { private static final Logger logger = LoggerFactory.getLogger(AbstractCacheFledgedQuery.class); + private static final String storageCacheTemplate = "StorageCache"; protected static CacheManager CACHE_MANAGER; @@ -37,31 +35,6 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup CACHE_MANAGER = cacheManager; } - /** - * This method is only useful non-spring injected test cases. - * When Kylin is normally ran as a spring app CACHE_MANAGER will be injected. - * and the configuration for cache lies in server/src/main/resources/ehcache.xml - * - * the cache named "StorageCache" acts like a template for each realization to - * create its own cache. - */ - private static void initCacheManger() { - Configuration conf = new Configuration(); - conf.setMaxBytesLocalHeap("128M"); - CACHE_MANAGER = CacheManager.create(conf); - - //a fake template for test cases - Cache storageCache = new Cache(new CacheConfiguration(storageCacheTemplate, 0).// - memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU).// - eternal(false).// - timeToIdleSeconds(86400).// - diskExpiryThreadIntervalSeconds(0).// - //maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).// - persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE))); - - CACHE_MANAGER.addCacheIfAbsent(storageCache); - } - protected StreamSQLResult getStreamSQLResult(StreamSQLDigest streamSQLDigest) { Cache cache = CACHE_MANAGER.getCache(this.underlyingStorage.getStorageUUID()); @@ -87,8 +60,7 @@ public abstract class AbstractCacheFledgedQuery implements IStorageQuery, TeeTup private void makeCacheIfNecessary(String storageUUID) { if (CACHE_MANAGER == null || (!(CACHE_MANAGER.getStatus().equals(Status.STATUS_ALIVE)))) { - logger.warn("CACHE_MANAGER is not provided or not alive"); - initCacheManger(); + throw new RuntimeException("CACHE_MANAGER is not provided or not alive"); } if (CACHE_MANAGER.getCache(storageUUID) == null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java index 53e5f5b..3193bbb 100644 --- a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java +++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java @@ -5,6 +5,7 @@ import java.util.IdentityHashMap; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import net.sf.ehcache.CacheManager; import org.apache.commons.lang.NotImplementedException; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.DateFormat; @@ -20,6 +21,7 @@ import org.apache.kylin.metadata.tuple.Tuple; import org.apache.kylin.metadata.tuple.TupleInfo; import org.apache.kylin.storage.ICachableStorageQuery; import org.apache.kylin.storage.StorageContext; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -32,10 +34,21 @@ import com.google.common.collect.Ranges; */ public class DynamicCacheTest { + private static CacheManager cacheManager; + @BeforeClass - public static void setup() { + public static void setupResource() throws Exception { System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); KylinConfig.getInstanceFromEnv().setProperty("kylin.query.cache.threshold.duration", "0"); + + cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); + AbstractCacheFledgedQuery.setCacheManager(cacheManager); + } + + @AfterClass + public static void tearDownResource() { + cacheManager.shutdown(); + AbstractCacheFledgedQuery.setCacheManager(null); } http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java index 182091b..b1665df 100644 --- a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java +++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java @@ -6,6 +6,7 @@ import java.util.IdentityHashMap; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import net.sf.ehcache.CacheManager; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.IdentityUtils; import org.apache.kylin.metadata.filter.TupleFilter; @@ -20,6 +21,7 @@ import org.apache.kylin.metadata.tuple.Tuple; import org.apache.kylin.metadata.tuple.TupleInfo; import org.apache.kylin.storage.ICachableStorageQuery; import org.apache.kylin.storage.StorageContext; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -30,12 +32,25 @@ import com.google.common.collect.Range; /** */ public class StaticCacheTest { + + private static CacheManager cacheManager; + @BeforeClass - public static void setup() { + public static void setupResource() throws Exception { + System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox"); - + KylinConfig config = KylinConfig.getInstanceFromEnv(); config.setProperty("kylin.query.cache.threshold.duration", "0"); + + cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); + AbstractCacheFledgedQuery.setCacheManager(cacheManager); + } + + @AfterClass + public static void tearDownResource() { + cacheManager.shutdown(); + AbstractCacheFledgedQuery.setCacheManager(null); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java index d6443e7..c253770 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java @@ -18,12 +18,14 @@ package org.apache.kylin.storage.hbase; -import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import net.sf.ehcache.CacheManager; + import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.apache.kylin.cube.CubeInstance; @@ -38,6 +40,7 @@ import org.apache.kylin.metadata.tuple.ITupleIterator; import org.apache.kylin.storage.IStorageQuery; import org.apache.kylin.storage.StorageContext; import org.apache.kylin.storage.StorageFactory; +import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery; import org.apache.kylin.storage.cache.StorageMockUtils; import org.apache.kylin.storage.exception.ScanOutOfLimitException; import org.junit.After; @@ -54,12 +57,18 @@ public class ITStorageTest extends HBaseMetadataTestCase { private CubeInstance cube; private StorageContext context; + private static CacheManager cacheManager; + @BeforeClass public static void setupResource() throws Exception { + cacheManager = CacheManager.newInstance("../server/src/main/resources/ehcache-test.xml"); + AbstractCacheFledgedQuery.setCacheManager(cacheManager); } @AfterClass public static void tearDownResource() { + cacheManager.shutdown(); + AbstractCacheFledgedQuery.setCacheManager(null); } @Before http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java index 294f399..01d3041 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java @@ -8,7 +8,7 @@ import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.kv.RowKeyColumnIO; import org.apache.kylin.dict.DictCodeSystem; import org.apache.kylin.dict.IDictionaryAware; -import org.apache.kylin.dict.TupleFilterDictionaryTranslater; +import org.apache.kylin.dict.TupleFilterFunctionTranslator; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; import org.apache.kylin.metadata.filter.ConstantTupleFilter; @@ -131,7 +131,7 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator { if (filter == null) return null; - ITupleFilterTranslator translator = new TupleFilterDictionaryTranslater(columnIO.getIDictionaryAware()); + ITupleFilterTranslator translator = new TupleFilterFunctionTranslator(columnIO.getIDictionaryAware()); filter = translator.translate(filter); // un-evaluatable filter is replaced with TRUE http://git-wip-us.apache.org/repos/asf/kylin/blob/3f5074ee/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java index abfb74d..e96c602 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java @@ -23,7 +23,7 @@ import org.apache.kylin.cube.gridtable.CubeGridTable; import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping; import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException; import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.dict.TupleFilterDictionaryTranslater; +import org.apache.kylin.dict.TupleFilterFunctionTranslator; import org.apache.kylin.gridtable.EmptyGTScanner; import org.apache.kylin.gridtable.GTInfo; import org.apache.kylin.gridtable.GTRecord; @@ -64,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner { CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping(); // translate FunctionTupleFilter to IN clause - ITupleFilterTranslator translator = new TupleFilterDictionaryTranslater(this.cubeSeg); + ITupleFilterTranslator translator = new TupleFilterFunctionTranslator(this.cubeSeg); filter = translator.translate(filter); //replace the constant values in filter to dictionary codes