Repository: kylin Updated Branches: refs/heads/master 51830456f -> 58991331d
KYLIN-2221 rethink on KYLIN-1684 Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58991331 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58991331 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58991331 Branch: refs/heads/master Commit: 58991331d276e127695893c2041bce37934c836f Parents: 5183045 Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Nov 22 13:42:46 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Nov 22 13:42:57 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 22 +++++++++++++------- .../gtrecord/GTCubeStorageQueryBase.java | 15 ++++++------- 2 files changed, 20 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/58991331/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index c7dd8a8..3252245 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -18,13 +18,6 @@ package org.apache.kylin.common; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.common.util.CliCommandExecutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -35,6 +28,14 @@ import java.util.SortedSet; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.util.CliCommandExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + /** * An abstract class to encapsulate access to a set of 'properties'. * Subclass can override methods in this class to extend the content of the 'properties', @@ -210,7 +211,7 @@ abstract public class KylinConfigBase implements Serializable { public String[] getRealizationProviders() { return getOptionalStringArray("kylin.realization.providers", // - new String[] {"org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager"}); + new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" }); } public CliCommandExecutor getCliCommandExecutor() throws IOException { @@ -490,6 +491,11 @@ abstract public class KylinConfigBase implements Serializable { return Float.parseFloat(getOptional("kylin.hbase.hfile.size.gb", "2.0")); } + //check KYLIN-1684, in most cases keep the default value + public boolean isSkippingEmptySegments() { + return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true")); + } + public int getStoragePushDownLimitMax() { return Integer.parseInt(getOptional("kylin.query.pushdown.limit.max", "10000")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/58991331/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index e715120..24dd2fb 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -125,16 +125,13 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { List<CubeSegmentScanner> scanners = Lists.newArrayList(); for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) { CubeSegmentScanner scanner; - try { - scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context, getGTStorage()); - } catch (IllegalArgumentException ex) { - // ref KYLIN-1967, real empty segment can trigger dictionary exception -- IllegalArgumentException: Value not exists! - if (cubeSeg.getInputRecords() == 0) { - logger.warn("cube segment {} input record is 0, skip it still", cubeSeg); - continue; - } - throw ex; + + if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) { + logger.info("Skip cube segment {} because its input record is 0", cubeSeg); + continue; } + + scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context, getGTStorage()); scanners.add(scanner); }
