Repository: kylin Updated Branches: refs/heads/KYLIN-2622 [created] 2551d489b
KYLIN-2622 Add SegmentAppendTrieDictBuilder for count distinct measure Signed-off-by: Li Yang <liy...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7530edb5 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7530edb5 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7530edb5 Branch: refs/heads/KYLIN-2622 Commit: 7530edb507b7f174e39413c24ed594d587c4fe99 Parents: 228b34a Author: kangkaisen <kangkai...@live.com> Authored: Tue May 16 17:04:55 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sat Sep 23 16:28:58 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/dict/AppendTrieDictionary.java | 9 ++- .../apache/kylin/dict/DictionaryManager.java | 10 +++ .../kylin/dict/GlobalDictionaryBuilder.java | 8 +-- .../global/AppendTrieDictionaryBuilder.java | 15 ++-- .../kylin/dict/global/GlobalDictHDFSStore.java | 54 +++++++++----- .../kylin/dict/global/GlobalDictMetadata.java | 1 - .../kylin/dict/global/GlobalDictStore.java | 5 +- .../global/SegmentAppendTrieDictBuilder.java | 76 ++++++++++++++++++++ .../dict/global/AppendTrieDictionaryTest.java | 3 +- .../kylin/measure/bitmap/BitmapMeasureType.java | 21 +----- .../localmeta/cube_desc/ci_inner_join_cube.json | 2 +- webapp/app/js/model/cubeConfig.js | 3 +- .../cubeDesigner/advanced_settings.html | 13 +++- 13 files changed, 162 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index b9f0d2b..18795b4 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -38,8 +38,8 @@ import java.io.IOException; import java.io.PrintStream; import java.util.Arrays; import java.util.Objects; +import java.util.TreeMap; import java.util.concurrent.ExecutionException; -import static com.google.common.base.Preconditions.checkState; /** * A dictionary based on Trie data structure that maps enumerations of byte[] to @@ -72,7 +72,12 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> { this.baseDir = baseDir; final GlobalDictStore globalDictStore = new GlobalDictHDFSStore(baseDir); Long[] versions = globalDictStore.listAllVersions(); - checkState(versions.length > 0, "Global dict at %s is empty", baseDir); + + if (versions.length == 0) { + this.metadata = new GlobalDictMetadata(0, 0, 0, 0, null, new TreeMap<AppendDictSliceKey, String>()); + return; // for the removed SegmentAppendTrieDictBuilder + } + final long latestVersion = versions[versions.length - 1]; final Path latestVersionPath = globalDictStore.getVersionDir(latestVersion); this.metadata = globalDictStore.getMetadata(latestVersion); http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java index 135ab0c..0eab8e4 100755 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java @@ -218,6 +218,16 @@ public class DictionaryManager { if (dicts.size() == 1) return dicts.get(0); + /** + * AppendTrieDictionary needn't merge + * more than one AppendTrieDictionary will generate when user use {@link SegmentAppendTrieDictBuilder} + */ + for (DictionaryInfo dict: dicts) { + if (dict.getDictionaryClass().equals(AppendTrieDictionary.class.getName())) { + return dict; + } + } + DictionaryInfo firstDictInfo = null; int totalSize = 0; for (DictionaryInfo info : dicts) { http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java index a593371..404d53c 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java @@ -44,16 +44,13 @@ public class GlobalDictionaryBuilder implements IDictionaryBuilder { @Override public void init(DictionaryInfo dictInfo, int baseId) throws IOException { - if (dictInfo == null) { - throw new IllegalArgumentException("GlobalDictinaryBuilder must used with an existing DictionaryInfo"); - } - sourceColumn = dictInfo.getSourceTable() + "_" + dictInfo.getSourceColumn(); lock = KylinConfig.getInstanceFromEnv().getDistributedLockFactory().lockForCurrentThread(); lock.lock(getLockPath(sourceColumn), Long.MAX_VALUE); int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize(); - this.builder = new AppendTrieDictionaryBuilder(dictInfo.getResourceDir(), maxEntriesPerSlice); + String baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict" + dictInfo.getResourceDir() + "/"; + this.builder = new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, true); this.baseId = baseId; } @@ -96,5 +93,4 @@ public class GlobalDictionaryBuilder implements IDictionaryBuilder { private String getLockPath(String pathName) { return "/dict/" + pathName + "/lock"; } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java index 90d65b6..54978c2 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java @@ -18,7 +18,6 @@ package org.apache.kylin.dict.global; -import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.dict.AppendTrieDictionary; import org.apache.kylin.dict.BytesConverter; @@ -35,6 +34,7 @@ public class AppendTrieDictionaryBuilder { private final String baseDir; private final String workingDir; private final int maxEntriesPerSlice; + private final boolean isAppendDictGlobal; private GlobalDictStore store; private int maxId; @@ -46,20 +46,21 @@ public class AppendTrieDictionaryBuilder { private AppendDictSliceKey curKey; private AppendDictNode curNode; - public AppendTrieDictionaryBuilder(String resourceDir, int maxEntriesPerSlice) throws IOException { - this.baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict" + resourceDir + "/"; - this.workingDir = this.baseDir + "/working"; + public AppendTrieDictionaryBuilder(String baseDir, int maxEntriesPerSlice, boolean isAppendDictGlobal) throws IOException { + this.baseDir = baseDir; + this.workingDir = baseDir + "working"; this.maxEntriesPerSlice = maxEntriesPerSlice; + this.isAppendDictGlobal = isAppendDictGlobal; init(); } public synchronized void init() throws IOException { this.store = new GlobalDictHDFSStore(baseDir); - store.prepareForWrite(workingDir); + store.prepareForWrite(workingDir, isAppendDictGlobal); Long[] versions = store.listAllVersions(); - if (versions.length == 0) { // build dict for the first time + if (versions.length == 0 || !isAppendDictGlobal) { // build dict for the first time this.maxId = 0; this.maxValueLength = 0; this.nValues = 0; @@ -122,7 +123,7 @@ public class AppendTrieDictionaryBuilder { } GlobalDictMetadata metadata = new GlobalDictMetadata(baseId, this.maxId, this.maxValueLength, this.nValues, this.bytesConverter, sliceFileMap); - store.commit(workingDir, metadata); + store.commit(workingDir, metadata, isAppendDictGlobal); AppendTrieDictionary dict = new AppendTrieDictionary(); dict.init(this.baseDir); http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java index b30d5b9..4c8ce0f 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java @@ -59,13 +59,6 @@ public class GlobalDictHDFSStore extends GlobalDictStore { this.basePath = new Path(baseDir); this.conf = HadoopUtil.getCurrentConfiguration(); this.fileSystem = HadoopUtil.getFileSystem(baseDir); - - if (!fileSystem.exists(basePath)) { - logger.info("Global dict at {} doesn't exist, create a new one", basePath); - fileSystem.mkdirs(basePath); - } - - migrateOldLayout(); } // Previously we put slice files and index file directly in base directory, @@ -111,8 +104,15 @@ public class GlobalDictHDFSStore extends GlobalDictStore { } @Override - void prepareForWrite(String workingDir) throws IOException { - // TODO create lock file + void prepareForWrite(String workingDir, boolean isGlobal) throws IOException { + if (!fileSystem.exists(basePath)) { + logger.info("Global dict at {} doesn't exist, create a new one", basePath); + fileSystem.mkdirs(basePath); + } + + migrateOldLayout(); + + logger.info("Prepare to write Global dict at {}, isGlobal={}", workingDir, isGlobal); Path working = new Path(workingDir); if (fileSystem.exists(working)) { @@ -122,7 +122,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore { // when build dict, copy all data into working dir and work on it, avoiding suddenly server crash made data corrupt Long[] versions = listAllVersions(); - if (versions.length > 0) { + if (versions.length > 0 && isGlobal) { Path latestVersion = getVersionDir(versions[versions.length - 1]); FileUtil.copy(fileSystem, latestVersion, fileSystem, working, false, true, conf); } else { @@ -132,6 +132,10 @@ public class GlobalDictHDFSStore extends GlobalDictStore { @Override public Long[] listAllVersions() throws IOException { + if (!fileSystem.exists(basePath)) { + return new Long[0]; // for the removed SegmentAppendTrieDictBuilder + } + FileStatus[] versionDirs = fileSystem.listStatus(basePath, new PathFilter() { @Override public boolean accept(Path path) { @@ -208,7 +212,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore { } @Override - public void commit(String workingDir, GlobalDictMetadata metadata) throws IOException { + public void commit(String workingDir, GlobalDictMetadata metadata, boolean isAppendDictGlobal) throws IOException { Path workingPath = new Path(workingDir); // delete v1 index file @@ -225,22 +229,38 @@ public class GlobalDictHDFSStore extends GlobalDictStore { Path newVersionPath = new Path(basePath, VERSION_PREFIX + System.currentTimeMillis()); fileSystem.rename(workingPath, newVersionPath); - cleanUp(); + cleanUp(isAppendDictGlobal); } // Check versions count, delete expired versions - private void cleanUp() throws IOException { - Long[] versions = listAllVersions(); + private void cleanUp(boolean isAppendDictGlobal) throws IOException { long timestamp = System.currentTimeMillis(); - for (int i = 0; i < versions.length - maxVersions; i++) { - if (versions[i] + versionTTL < timestamp) { - fileSystem.delete(getVersionDir(versions[i]), true); + if (isAppendDictGlobal) { + Long[] versions = listAllVersions(); + for (int i = 0; i < versions.length - maxVersions; i++) { + if (versions[i] + versionTTL < timestamp) { + fileSystem.delete(getVersionDir(versions[i]), true); + } + } + } else { + FileStatus[] segmentDictDirs = fileSystem.listStatus(basePath.getParent()); + for (FileStatus fileStatus : segmentDictDirs) { + String filePath = fileStatus.getPath().getName(); + Long version = Long.parseLong(filePath.split("_")[1]); + if (version + versionTTL < timestamp) { + fileSystem.delete(new Path(basePath.getParent() + "/" + filePath), true); + } } } } @Override public String copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException { + if (baseDir.contains("resources/SegmentDict")) { + logger.info("SegmentAppendTrieDict needn't to copy"); + return baseDir; + } + checkArgument(baseDir.startsWith(srcConfig.getHdfsWorkingDirectory()), "Please check why current directory {} doesn't belong to source working directory {}", baseDir, srcConfig.getHdfsWorkingDirectory()); final String dstBaseDir = baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()); http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java index 7c89ea2..7fe6b60 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java @@ -38,7 +38,6 @@ public class GlobalDictMetadata { public GlobalDictMetadata(int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter, NavigableMap<AppendDictSliceKey, String> sliceFileMap) { - Preconditions.checkNotNull(bytesConverter, "bytesConverter"); Preconditions.checkNotNull(sliceFileMap, "sliceFileMap"); this.baseId = baseId; http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java index eaf0729..f686680 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java @@ -37,7 +37,7 @@ public abstract class GlobalDictStore { } // workingDir should be an absolute path, will create if not exists - abstract void prepareForWrite(String workingDir) throws IOException; + abstract void prepareForWrite(String workingDir, boolean isGlobal) throws IOException; /** * @return all versions of this dictionary in ascending order @@ -87,9 +87,10 @@ public abstract class GlobalDictStore { * commit the <i>DictSlice</i> and <i>GlobalDictMetadata</i> in workingDir to new versionDir * @param workingDir where store the tmp slice and index, should exist * @param globalDictMetadata the metadata of global dict + * @param isAppendDictGlobal mark the append dict whether is global or not * @throws IOException on I/O error */ - public abstract void commit(String workingDir, GlobalDictMetadata globalDictMetadata) throws IOException; + public abstract void commit(String workingDir, GlobalDictMetadata globalDictMetadata, boolean isAppendDictGlobal) throws IOException; /** * Copy the latest version of this dict to another meta. The source is unchanged. http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java new file mode 100644 index 0000000..270deee --- /dev/null +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.dict.global; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.Dictionary; +import org.apache.kylin.dict.DictionaryInfo; +import org.apache.kylin.dict.IDictionaryBuilder; + +import java.io.IOException; +import java.util.UUID; + +/** + * SegmentAppendTrieDictBuilder based on one segment. + * SegmentAppendTrieDictBuilder only used for count distinct measure that needn't rollup among segments. + * SegmentAppendTrieDictBuilder could avoid AppendTrieDictionary infinite growth. + * SegmentAppendTrieDictBuilder doesn't support merge. + */ +public class SegmentAppendTrieDictBuilder implements IDictionaryBuilder { + private AppendTrieDictionaryBuilder builder; + private int baseId; + private String sourceColumn; + + @Override + public void init(DictionaryInfo dictInfo, int baseId) throws IOException { + sourceColumn = dictInfo.getSourceTable() + "." + dictInfo.getSourceColumn(); + + int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize(); + if (hdfsDir == null) { + //build in Kylin job server + hdfsDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(); + } + //use UUID to make each segment dict in different HDFS dir and support concurrent build + //use timestamp to make the segment dict easily to delete + String baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/SegmentDict" + dictInfo.getResourceDir() + "/" + UUID.randomUUID().toString() + "_" + System.currentTimeMillis()+ "/"; + + this.builder = new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, false); + this.baseId = baseId; + } + + @Override + public boolean addValue(String value) { + if (value == null) { + return false; + } + + try { + builder.addValue(value); + } catch (Throwable e) { + throw new RuntimeException(String.format("Failed to create global dictionary on %s ", sourceColumn), e); + } + + return true; + } + + @Override + public Dictionary<String> build() throws IOException { + return builder.build(baseId); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java index 47011fe..6b39c36 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java @@ -94,7 +94,8 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { private AppendTrieDictionaryBuilder createBuilder(String resourceDir) throws IOException { int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize(); - return new AppendTrieDictionaryBuilder(resourceDir, maxEntriesPerSlice); + String baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + resourceDir + "/"; + return new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, true); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java index e4fb079..403d1b6 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java @@ -126,25 +126,8 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> { @Override public BitmapCounter reEncodeDictionary(BitmapCounter value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) { - if (!needDictionaryColumn(measureDesc.getFunction())) { - return value; - } - TblColRef colRef = measureDesc.getFunction().getParameter().getColRefs().get(0); - Dictionary<String> sourceDict = oldDicts.get(colRef); - Dictionary<String> mergedDict = newDicts.get(colRef); - - BitmapCounter retValue = factory.newBitmap(); - for (int id : value) { - int newId; - String v = sourceDict.getValueFromId(id); - if (v == null) { - newId = mergedDict.nullId(); - } else { - newId = mergedDict.getIdFromValue(v); - } - retValue.add(newId); - } - return retValue; + //BitmapCounter needn't reEncode + return value; } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json index cfd8cd1..4346206 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json @@ -359,7 +359,7 @@ "dictionaries": [ { "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP", - "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" + "builder": "org.apache.kylin.dict.global.SegmentAppendTrieDictBuilder" } ], "rowkey": { http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/webapp/app/js/model/cubeConfig.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js index eec3f10..7caa19b 100644 --- a/webapp/app/js/model/cubeConfig.js +++ b/webapp/app/js/model/cubeConfig.js @@ -110,7 +110,8 @@ KylinApp.constant('cubeConfig', { ], statusNeedNofity:['ERROR', 'DISCARDED', 'SUCCEED'], buildDictionaries:[ - {name:"Global Dictionary", value:"org.apache.kylin.dict.GlobalDictionaryBuilder"} + {name:"Global Dictionary", value:"org.apache.kylin.dict.GlobalDictionaryBuilder"}, + {name:"Segment Dictionary", value:"org.apache.kylin.dict.global.SegmentAppendTrieDictBuilder"} ], needSetLengthEncodingList:['fixed_length','fixed_length_hex','int','integer'] }); http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/webapp/app/partials/cubeDesigner/advanced_settings.html ---------------------------------------------------------------------- diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html index 8062975..8768108 100755 --- a/webapp/app/partials/cubeDesigner/advanced_settings.html +++ b/webapp/app/partials/cubeDesigner/advanced_settings.html @@ -534,7 +534,18 @@ </script> <script type="text/ng-template" id="AdvancedDictionariesTip.html"> <div> - <h4>Special settings for dictionaries. Leave blank by default.</h4> + <h4>Special settings for dictionaries.</h4> + <ol> + <li> + "Global Dictionary" is the default dict for precise count distinct measure, which support rollup among all segments. + </li> + <li> + "Segment Dictionary" is the special dict for precise count distinct measure, which is based on one segment and could not + support rollup among segments. + Specifically, if your cube isn't partitioned or you can ensure all your SQLs will group by your partition_column, + you could use "Segment Dictionary" instead of "Global Dictionary". + </li> + </ol> </div> </script>
