KYLIN-2414 minor rename on properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3fff6d1 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3fff6d1 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3fff6d1 Branch: refs/heads/KYLIN-2361 Commit: c3fff6d19d355e78461fa7f32d02feabc5bf63c8 Parents: e6a9382 Author: shaofengshi <shaofeng...@apache.org> Authored: Thu Jan 26 10:53:19 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Thu Jan 26 10:53:19 2017 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 8 +++++++- .../org/apache/kylin/common/KylinConfigBase.java | 4 ++-- .../mr/steps/FactDistinctColumnsReducer.java | 19 +++++++++---------- 3 files changed, 18 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 43ea17d..1232c47 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -127,9 +127,15 @@ kylin.engine.mr.max-reducer-number=500 kylin.engine.mr.mapper-input-rows=1000000 +# Enable dictionary building in MR reducer +kylin.engine.mr.build-dict-in-reducer=true + +# Number of reducers for fetching UHC column distinct values +kylin.engine.mr.uhc-reducer-count=1 + ### CUBE | DICTIONARY ### -# 'auto', 'inmem', 'layer' or 'random' for testing +# 'auto', 'inmem' or 'layer' kylin.cube.algorithm=auto # A smaller threshold prefers layer, a larger threshold prefers in-mem http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b25bcc0..6a88fc4 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -740,8 +740,8 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1")); } - public boolean isReducerLocalBuildDict() { - return Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", "true")); + public boolean isBuildDictInReducerEnabled() { + return Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer", "true")); } public String getYarnStatusCheckUrl() { http://git-wip-us.apache.org/repos/asf/kylin/blob/c3fff6d1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java index cf94b30..5b795c2 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java @@ -78,7 +78,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK private int rowCount = 0; //local build dict - private boolean isReducerLocalBuildDict; + private boolean buildDictInReducer; private IDictionaryBuilder builder; private long timeMaxValue = Long.MIN_VALUE; private long timeMinValue = Long.MAX_VALUE; @@ -119,30 +119,29 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK isPartitionCol = true; col = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef(); if (col == null) { - logger.info("Do not have partition col. This reducer will keep empty"); + logger.info("No partition col. This reducer will do nothing"); } } else { // normal col col = columnList.get(reducerIdToColumnIndex.get(taskId)); - Preconditions.checkNotNull(col); // local build dict - isReducerLocalBuildDict = config.isReducerLocalBuildDict(); + buildDictInReducer = config.isBuildDictInReducerEnabled(); if (cubeDesc.getDictionaryBuilderClass(col) != null) { // only works with default dictionary builder - isReducerLocalBuildDict = false; + buildDictInReducer = false; } if(config.getUHCReducerCount() > 1) { int[] uhcIndex = CubeManager.getInstance(config).getUHCIndex(cubeDesc); int colIndex = reducerIdToColumnIndex.get(taskId); if (uhcIndex[colIndex] == 1) - isReducerLocalBuildDict = false; //for UHC columns, this feature should be disabled + buildDictInReducer = false; //for UHC columns, this feature should be disabled } - if (isReducerLocalBuildDict) { + if (buildDictInReducer) { builder = DictionaryGenerator.newDictionaryBuilder(col.getType()); builder.init(null, 0); } - logger.info("Reducer " + taskId + " handling column " + col + ", isReducerLocalBuildDict=" + isReducerLocalBuildDict); + logger.info("Reducer " + taskId + " handling column " + col + ", buildDictInReducer=" + buildDictInReducer); } } @@ -192,7 +191,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK timeMaxValue = Math.max(timeMaxValue, time); } else { // normal col - if (isReducerLocalBuildDict) { + if (buildDictInReducer) { String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1); logAFewRows(value); builder.addValue(value); @@ -228,7 +227,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK outputPartitionInfo(); } else { // normal col - if (isReducerLocalBuildDict) { + if (buildDictInReducer) { Dictionary<String> dict = builder.build(); outputDict(col, dict); }