kylin git commit: KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable

2016-04-14 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master 5042a30bc -> f3a33673f


KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f3a33673
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f3a33673
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f3a33673

Branch: refs/heads/master
Commit: f3a33673fcc39e3673a1b1e712fb44fcf781d8f5
Parents: 5042a30
Author: Hongbin Ma 
Authored: Thu Apr 14 15:06:31 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 14 15:06:31 2016 +0800

--
 .../kylin/gridtable/GTScanRangePlanner.java | 20 
 1 file changed, 12 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f3a33673/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 479064c..baf3428 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -63,9 +63,8 @@ public class GTScanRangePlanner {
 
 private static final Logger logger = 
LoggerFactory.getLogger(GTScanRangePlanner.class);
 
-private static final int MAX_HBASE_FUZZY_KEYS = 100;
-
 protected int maxScanRanges;
+protected int maxFuzzyKeys;
 
 //non-GT
 protected CubeSegment cubeSegment;
@@ -93,7 +92,8 @@ public class GTScanRangePlanner {
 public GTScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, 
TupleFilter filter, Set dimensions, Set groupbyDims, //
 Collection metrics) {
 
-this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
+this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
+this.maxFuzzyKeys = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
 
 this.cubeSegment = cubeSegment;
 this.cubeDesc = cubeSegment.getCubeDesc();
@@ -143,7 +143,9 @@ public class GTScanRangePlanner {
  */
 public GTScanRangePlanner(GTInfo info, Pair 
gtStartAndEnd, TblColRef gtPartitionCol, TupleFilter gtFilter) {
 
-this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
+this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax();
+this.maxFuzzyKeys = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
+
 this.gtInfo = info;
 
 IGTComparator comp = gtInfo.codeSystem.getComparator();
@@ -154,11 +156,13 @@ public class GTScanRangePlanner {
 //start key GTRecord compare to stop key GTRecord
 this.rangeStartEndComparator = getRangeStartEndComparator(comp);
 
+
 this.gtFilter = gtFilter;
 this.gtStartAndEnd = gtStartAndEnd;
 this.gtPartitionCol = gtPartitionCol;
 }
 
+
 public GTScanRequest planScanRequest(boolean allowPreAggregate) {
 GTScanRequest scanRequest;
 List scanRanges = this.planScanRanges();
@@ -308,8 +312,8 @@ public class GTScanRangePlanner {
 if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) 
{
 if 
(rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), 
range.end) <= 0 //
 && 
(rangeStartEndComparator.comparator.compare(range.begin, 
gtStartAndEnd.getSecond()) < 0 //
-|| 
rangeStartEndComparator.comparator.compare(range.begin, 
gtStartAndEnd.getSecond()) == 0 //
-&& (range.op == FilterOperatorEnum.EQ 
|| range.op == FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || 
range.op == FilterOperatorEnum.IN))) {
+|| 
rangeStartEndComparator.comparator.compare(range.begin, 
gtStartAndEnd.getSecond()) == 0 //
+&& (range.op == FilterOperatorEnum.EQ || range.op == 
FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || range.op == 
FilterOperatorEnum.IN))) {
 //segment range is [Closed,Open), but 
segmentStartAndEnd.getSecond() might be rounded, so use <= when has equals in 
condition. 
 } else {
 logger.debug("Pre-check partition col filter failed, 
partitionColRef {}, segment start {}, segment end {}, range begin {}, range end 
{}", //
@@ -346,7 +350,7 @@ public class GTScanRangePlanner {
 return result;
 }
 
-List>

kylin git commit: KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable

2016-04-13 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/master 5dfda003e -> 5042a30bc


KYLIN-1585 make MAX_HBASE_FUZZY_KEYS in GTScanRangePlanner configurable


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5042a30b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5042a30b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5042a30b

Branch: refs/heads/master
Commit: 5042a30bcbc51a1061d2c09365c994dceb28d44e
Parents: 5dfda00
Author: Hongbin Ma 
Authored: Thu Apr 14 12:17:31 2016 +0800
Committer: Hongbin Ma 
Committed: Thu Apr 14 12:17:31 2016 +0800

--
 .../main/java/org/apache/kylin/common/KylinConfigBase.java   | 4 
 .../java/org/apache/kylin/gridtable/GTScanRangePlanner.java  | 8 +++-
 2 files changed, 7 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/5042a30b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 712bcf7..51aa8aa 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -488,6 +488,10 @@ abstract public class KylinConfigBase implements 
Serializable {
 setProperty("kylin.query.storage.visit.planner", v);
 }
 
+public int getQueryScanFuzzyKeyMax() {
+return 
Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max","200"));
+}
+
 public int getQueryStorageVisitScanRangeMax() {
 return 
Integer.valueOf(this.getOptional("kylin.query.storage.visit.scanrange.max", 
"100"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/5042a30b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
--
diff --git 
a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java 
b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 3f9bac0..479064c 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -32,6 +32,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.DateFormat;
@@ -62,7 +63,6 @@ public class GTScanRangePlanner {
 
 private static final Logger logger = 
LoggerFactory.getLogger(GTScanRangePlanner.class);
 
-private static final int MAX_SCAN_RANGES = 200;
 private static final int MAX_HBASE_FUZZY_KEYS = 100;
 
 protected int maxScanRanges;
@@ -93,7 +93,7 @@ public class GTScanRangePlanner {
 public GTScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, 
TupleFilter filter, Set dimensions, Set groupbyDims, //
 Collection metrics) {
 
-this.maxScanRanges = MAX_SCAN_RANGES;
+this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
 
 this.cubeSegment = cubeSegment;
 this.cubeDesc = cubeSegment.getCubeDesc();
@@ -143,7 +143,7 @@ public class GTScanRangePlanner {
  */
 public GTScanRangePlanner(GTInfo info, Pair 
gtStartAndEnd, TblColRef gtPartitionCol, TupleFilter gtFilter) {
 
-this.maxScanRanges = MAX_SCAN_RANGES;
+this.maxScanRanges = 
KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax();
 this.gtInfo = info;
 
 IGTComparator comp = gtInfo.codeSystem.getComparator();
@@ -154,12 +154,10 @@ public class GTScanRangePlanner {
 //start key GTRecord compare to stop key GTRecord
 this.rangeStartEndComparator = getRangeStartEndComparator(comp);
 
-
 this.gtFilter = gtFilter;
 this.gtStartAndEnd = gtStartAndEnd;
 this.gtPartitionCol = gtPartitionCol;
 }
-
 
 public GTScanRequest planScanRequest(boolean allowPreAggregate) {
 GTScanRequest scanRequest;