kylin git commit: KYLIN-2221 rethink on KYLIN-1684

[shaofengshi]

Repository: kylin
Updated Branches:
  refs/heads/v1.6.0-rc2 783d39917 -> 33bb7c293


KYLIN-2221 rethink on KYLIN-1684


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/33bb7c29
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/33bb7c29
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/33bb7c29

Branch: refs/heads/v1.6.0-rc2
Commit: 33bb7c293982a052a8c436820c708a038878e30f
Parents: 783d399
Author: Hongbin Ma 
Authored: Tue Nov 22 13:42:46 2016 +0800
Committer: shaofengshi 
Committed: Tue Nov 22 15:59:17 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  5 
 .../gtrecord/GTCubeStorageQueryBase.java| 26 
 2 files changed, 10 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/33bb7c29/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 011155f..c2595fb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -477,6 +477,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Float.parseFloat(getOptional("kylin.hbase.hfile.size.gb", 
"2.0"));
 }
 
+//check KYLIN-1684, in most cases keep the default value
+public boolean isSkippingEmptySegments() {
+return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
+}
+
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.pushdown.limit.max", 
"1"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/33bb7c29/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 76a8c7b..5e8af88 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -122,24 +122,12 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 List scanners = Lists.newArrayList();
 for (CubeSegment cubeSeg : 
cubeInstance.getSegments(SegmentStatusEnum.READY)) {
 CubeSegmentScanner scanner;
-if (cubeSeg.getInputRecords() == 0) {
-if (!skipZeroInputSegment(cubeSeg)) {
-logger.warn("cube segment {} input record is 0, " + "it 
may caused by kylin failed to get the job counter " + "as the hadoop history 
server wasn't running", cubeSeg);
-} else {
-logger.warn("cube segment {} input record is 0, skip it ", 
cubeSeg);
-continue;
-}
-}
-try {
-scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context, getGTStorage());
-} catch (IllegalArgumentException ex) {
-// ref KYLIN-1967, real empty segment can trigger dictionary 
exception -- IllegalArgumentException: Value not exists!
-if (cubeSeg.getInputRecords() == 0) {
-logger.warn("cube segment {} input record is 0, skip it 
still", cubeSeg);
-continue;
-}
-throw ex;
+if (cubeDesc.getConfig().isSkippingEmptySegments() && 
cubeSeg.getInputRecords() == 0) {
+logger.info("Skip cube segment {} because its input record is 
0", cubeSeg);
+continue;
 }
+
+scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context, getGTStorage());
 scanners.add(scanner);
 }
 
@@ -149,10 +137,6 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, 
metrics, returnTupleInfo, context);
 }
 
-protected boolean skipZeroInputSegment(CubeSegment cubeSegment) {
-return false;
-}
-
 protected abstract String getGTStorage();
 
 private void buildDimensionsAndMetrics(SQLDigest sqlDigest, 
Collection dimensions, Collection metrics) {

kylin git commit: KYLIN-2221 rethink on KYLIN-1684

[mahongbin]

Repository: kylin
Updated Branches:
  refs/heads/master 51830456f -> 58991331d


KYLIN-2221 rethink on KYLIN-1684


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58991331
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58991331
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58991331

Branch: refs/heads/master
Commit: 58991331d276e127695893c2041bce37934c836f
Parents: 5183045
Author: Hongbin Ma 
Authored: Tue Nov 22 13:42:46 2016 +0800
Committer: Hongbin Ma 
Committed: Tue Nov 22 13:42:57 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java| 22 +---
 .../gtrecord/GTCubeStorageQueryBase.java| 15 ++---
 2 files changed, 20 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/58991331/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index c7dd8a8..3252245 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -18,13 +18,6 @@
 
 package org.apache.kylin.common;
 
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
@@ -35,6 +28,14 @@ import java.util.SortedSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
 /**
  * An abstract class to encapsulate access to a set of 'properties'.
  * Subclass can override methods in this class to extend the content of the 
'properties',
@@ -210,7 +211,7 @@ abstract public class KylinConfigBase implements 
Serializable {
 
 public String[] getRealizationProviders() {
 return getOptionalStringArray("kylin.realization.providers", //
-new String[] {"org.apache.kylin.cube.CubeManager", 
"org.apache.kylin.storage.hybrid.HybridManager"});
+new String[] { "org.apache.kylin.cube.CubeManager", 
"org.apache.kylin.storage.hybrid.HybridManager" });
 }
 
 public CliCommandExecutor getCliCommandExecutor() throws IOException {
@@ -490,6 +491,11 @@ abstract public class KylinConfigBase implements 
Serializable {
 return Float.parseFloat(getOptional("kylin.hbase.hfile.size.gb", 
"2.0"));
 }
 
+//check KYLIN-1684, in most cases keep the default value
+public boolean isSkippingEmptySegments() {
+return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", 
"true"));
+}
+
 public int getStoragePushDownLimitMax() {
 return Integer.parseInt(getOptional("kylin.query.pushdown.limit.max", 
"1"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/58991331/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index e715120..24dd2fb 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -125,16 +125,13 @@ public abstract class GTCubeStorageQueryBase implements 
IStorageQuery {
 List scanners = Lists.newArrayList();
 for (CubeSegment cubeSeg : 
cubeInstance.getSegments(SegmentStatusEnum.READY)) {
 CubeSegmentScanner scanner;
-try {
-scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, 
groupsD, metrics, filterD, context, getGTStorage());
-} catch (IllegalArgumentException ex) {
-// ref KYLIN-1967, real empty segment can trigger dictionary 
exception -- IllegalArgumentException: Value not exists!
-if (cubeSeg.getInputRecords() == 0) {
-logger.warn("cube segment {} input record is 0, skip it 
still", cubeSeg);
-continue;
-}
-throw ex;
+
+if