Repository: kylin Updated Branches: refs/heads/yang22-hbase1.x c33fa8c9d -> 7f82104e9 (forced update)
KYLIN-2290 minor improvements on limit Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cdd945cb Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cdd945cb Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cdd945cb Branch: refs/heads/yang22-hbase1.x Commit: cdd945cbe3c6a3326967dda4623bb4f3df855ef1 Parents: ed497fe Author: Hongbin Ma <mahong...@apache.org> Authored: Fri Dec 16 16:28:18 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Fri Dec 16 16:28:18 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 1 + .../kylin/cube/CubeCapabilityChecker.java | 5 ++ .../kylin/gridtable/GTScanRequestBuilder.java | 2 +- .../kylin/metadata/realization/SQLDigest.java | 4 +- .../apache/kylin/storage/StorageContext.java | 74 +++++++------------- .../storage/gtrecord/CubeScanRangePlanner.java | 9 +-- .../gtrecord/GTCubeStorageQueryBase.java | 1 - .../gtrecord/SequentialCubeTupleIterator.java | 6 +- .../apache/kylin/query/ITKylinQueryTest.java | 1 - .../org/apache/kylin/query/KylinTestBase.java | 7 +- .../kylin/storage/hbase/ITStorageTest.java | 2 +- .../apache/kylin/query/relnode/OLAPContext.java | 6 +- .../kylin/query/relnode/OLAPFilterRel.java | 2 +- .../kylin/query/relnode/OLAPLimitRel.java | 10 ++- .../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +- .../cube/v1/SerializedHBaseTupleIterator.java | 2 +- .../coprocessor/endpoint/CubeVisitService.java | 7 +- 17 files changed, 62 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 9923bc1..a1c9050 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -782,6 +782,7 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true")); } + @Deprecated//Limit is good even it's large. This config is meaning less since we already have scan threshold public int getStoragePushDownLimitMax() { return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java index 38faed9..c45144b 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java @@ -120,6 +120,11 @@ public class CubeCapabilityChecker { return result; } + if (digest.limitPrecedesAggr) { + logger.info("Exclude cube " + cube.getName() + " because there's limit preceding aggregation"); + return result; + } + if (digest.isRawQuery && rootFactTable.equals(digest.factTable)) { result.influences.add(new CapabilityInfluence() { @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java index f542de1..bcec1f4 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java @@ -36,7 +36,7 @@ public class GTScanRequestBuilder { private boolean allowStorageAggregation = true; private double aggCacheMemThreshold = 0; private int storageScanRowNumThreshold = Integer.MAX_VALUE;// storage should terminate itself when $storageScanRowNumThreshold cuboid rows are scanned, and throw exception. - private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit working when $toragePushDownLimit aggregated rows are produced. + private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit scanning safely when $toragePushDownLimit aggregated rows are produced. private long startTime = -1; private long timeout = -1; private String storageBehavior = null; http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java index 83fc05c..4780487 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java @@ -59,10 +59,11 @@ public class SQLDigest { public List<TblColRef> sortColumns; public List<OrderEnum> sortOrders; public boolean isRawQuery; + public boolean limitPrecedesAggr; public SQLDigest(String factTable, TupleFilter filter, List<JoinDesc> joinDescs, Set<TblColRef> allColumns, // List<TblColRef> groupbyColumns, Set<TblColRef> subqueryJoinParticipants, Set<TblColRef> filterColumns, Set<TblColRef> metricColumns, // - List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef> sortColumns, List<OrderEnum> sortOrders) { + List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef> sortColumns, List<OrderEnum> sortOrders,boolean limitPrecedesAggr) { this.factTable = factTable; this.filter = filter; this.joinDescs = joinDescs; @@ -76,6 +77,7 @@ public class SQLDigest { this.sortColumns = sortColumns; this.sortOrders = sortOrders; this.isRawQuery = isRawQuery(); + this.limitPrecedesAggr = limitPrecedesAggr; } private boolean isRawQuery() { http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java index bc43a87..9ef59fd 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java @@ -34,43 +34,29 @@ import com.google.common.collect.Range; public class StorageContext { private static final Logger logger = LoggerFactory.getLogger(StorageContext.class); - public static final int DEFAULT_THRESHOLD = 1000000; - private String connUrl; private int threshold; - private int limit; - private int offset; - private int finalPushDownLimit; - private boolean hasSort; - private boolean acceptPartialResult; - - private boolean exactAggregation; - private boolean needStorageAggregation; - private boolean enableLimit; - private boolean enableCoprocessor; - - private AtomicLong totalScanCount; + private int limit = Integer.MAX_VALUE; + private int offset = 0; + private int finalPushDownLimit = Integer.MAX_VALUE; + private boolean hasSort = false; + private boolean acceptPartialResult = false; + + private boolean exactAggregation = false; + private boolean needStorageAggregation = false; + private boolean limitEnabled = false; + private boolean enableCoprocessor = false; + + private AtomicLong totalScanCount = new AtomicLong(); private Cuboid cuboid; - private boolean partialResultReturned; - - private Range<Long> reusedPeriod; + private boolean partialResultReturned = false; public StorageContext() { - this.threshold = DEFAULT_THRESHOLD; - this.limit = DEFAULT_THRESHOLD; - this.totalScanCount = new AtomicLong(); - this.cuboid = null; - this.hasSort = false; - - this.exactAggregation = false; - this.enableLimit = false; - this.enableCoprocessor = false; - - this.acceptPartialResult = false; - this.partialResultReturned = false; - this.finalPushDownLimit = Integer.MAX_VALUE; + this.threshold = KylinConfig.getInstanceFromEnv().getScanThreshold(); } + private Range<Long> reusedPeriod; + public String getConnUrl() { return connUrl; } @@ -92,11 +78,10 @@ public class StorageContext { } public void setLimit(int l) { - if (l > limit) { - //cases like : select price from (select * from kylin_sales limit 10) limit 5000 - logger.info("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, limit); + if (limit != Integer.MAX_VALUE) { + logger.warn("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, limit); } else { - this.limit = l; + limit = l; } } @@ -109,15 +94,11 @@ public class StorageContext { } public void enableLimit() { - this.enableLimit = true; + this.limitEnabled = true; } public boolean isLimitEnabled() { - return this.enableLimit; - } - - private int getStoragePushDownLimit() { - return this.isLimitEnabled() ? this.getOffset() + this.getLimit() : Integer.MAX_VALUE; + return this.limitEnabled; } public int getFinalPushDownLimit() { @@ -126,19 +107,16 @@ public class StorageContext { public void setFinalPushDownLimit(IRealization realization) { - //decide the final limit push down - int tempPushDownLimit = this.getStoragePushDownLimit(); - if (tempPushDownLimit == Integer.MAX_VALUE) { + if (this.getLimit() == Integer.MAX_VALUE) { return; } - int pushDownLimitMax = KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax(); + int tempPushDownLimit = this.getOffset() + this.getLimit(); + if (!realization.supportsLimitPushDown()) { - logger.info("Not enabling limit push down because cube storage type not supported"); - } else if (tempPushDownLimit > pushDownLimitMax) { - logger.info("Not enabling limit push down because the limit(including offset) {} is larger than kylin.query.max-limit-pushdown {}", // - tempPushDownLimit, pushDownLimitMax); + logger.warn("Not enabling limit push down because cube storage type not supported"); } else { + this.limitEnabled = true; this.finalPushDownLimit = tempPushDownLimit; logger.info("Enable limit: " + tempPushDownLimit); } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java index 8d5a3d4..b05a629 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java @@ -148,15 +148,10 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase { GTScanRequest scanRequest; List<GTScanRange> scanRanges = this.planScanRanges(); if (scanRanges != null && scanRanges.size() != 0) { - GTScanRequestBuilder builder = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).// + scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).// setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs).setFilterPushDown(gtFilter).// setAllowStorageAggregation(context.isNeedStorageAggregation()).setAggCacheMemThreshold(cubeSegment.getCubeInstance().getConfig().getQueryCoprocessorMemGB()).// - setStorageScanRowNumThreshold(context.getThreshold()); - - if (context.getFinalPushDownLimit() != Integer.MAX_VALUE) - builder.setStoragePushDownLimit(context.getFinalPushDownLimit()); - - scanRequest = builder.createGTScanRequest(); + setStoragePushDownLimit(context.getFinalPushDownLimit()).setStorageScanRowNumThreshold(context.getThreshold()).createGTScanRequest(); } else { scanRequest = null; } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index 9c74cca..85de844 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -383,7 +383,6 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { } if (possible) { - context.enableLimit(); context.setFinalPushDownLimit(cubeInstance); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java index bef0e88..3a64de7 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java @@ -46,7 +46,6 @@ public class SequentialCubeTupleIterator implements ITupleIterator { protected List<CubeSegmentScanner> scanners; protected List<SegmentCubeTupleIterator> segmentCubeTupleIterators; protected Iterator<ITuple> tupleIterator; - protected final int storagePushDownLimit; protected StorageContext context; private int scanCount; @@ -62,8 +61,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator { segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context)); } - this.storagePushDownLimit = context.getFinalPushDownLimit(); - if (storagePushDownLimit == Integer.MAX_VALUE) { + if (!context.isLimitEnabled()) { //normal case tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator()); } else { @@ -75,7 +73,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator { return input; } }); - tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, storagePushDownLimit, segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator(); + tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 0379bd5..98b294d 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -338,7 +338,6 @@ public class ITKylinQueryTest extends KylinTestBase { List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql"); for (File sqlFile : sqlFiles) { runSQL(sqlFile, false, false); - assertTrue(checkLimitEnabled()); assertTrue(checkFinalPushDownLimit()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index 114a28d..b8a48ef 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -655,14 +655,9 @@ public class KylinTestBase { } - protected boolean checkLimitEnabled() { - OLAPContext context = getFirstOLAPContext(); - return (context.storageContext.isLimitEnabled()); - } - protected boolean checkFinalPushDownLimit() { OLAPContext context = getFirstOLAPContext(); - return (context.storageContext.getFinalPushDownLimit() != Integer.MAX_VALUE); + return context.storageContext.isLimitEnabled(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java index db9d133..aea8bef 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java @@ -148,7 +148,7 @@ public class ITStorageTest extends HBaseMetadataTestCase { int count = 0; ITupleIterator iterator = null; try { - SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>()); + SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false); iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations)); while (iterator.hasNext()) { ITuple tuple = iterator.next(); http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java index 8278fb0..dde98a6 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java @@ -112,7 +112,9 @@ public class OLAPContext { public Set<OLAPTableScan> allTableScans = new HashSet<>(); public TupleInfo returnTupleInfo = null; public boolean afterAggregate = false; - public boolean afterSkippedFilter = false; + public boolean afterHavingClauseFilter = false; + public boolean afterLimit = false; + public boolean limitPrecedesAggr = false; public boolean afterJoin = false; public boolean hasJoin = false; @@ -148,7 +150,7 @@ public class OLAPContext { public SQLDigest getSQLDigest() { if (sqlDigest == null) - sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortColumns, sortOrders); + sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortColumns, sortOrders, limitPrecedesAggr); return sqlDigest; } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java index 411142d..1981c10 100755 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java @@ -302,7 +302,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel { if (!context.afterAggregate) { translateFilter(context); } else { - context.afterSkippedFilter = true;//having clause is skipped + context.afterHavingClauseFilter = true;//having clause is skipped } } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java index f0af863..9ebdf60 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java @@ -77,7 +77,9 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel { this.columnRowType = buildColumnRowType(); this.context = implementor.getContext(); - if (!context.afterSkippedFilter) { + // ignore limit after having clause + // ignore limit after another limit, e.g. select A, count(*) from (select A,B from fact group by A,B limit 100) limit 10 + if (!context.afterHavingClauseFilter && !context.afterLimit) { Number limitValue = (Number) (((RexLiteral) localFetch).getValue()); int limit = limitValue.intValue(); this.context.storageContext.setLimit(limit); @@ -87,6 +89,12 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel { int offset = offsetValue.intValue(); this.context.storageContext.setOffset(offset); } + + context.afterLimit = true; + + if (!this.context.afterAggregate) { + this.context.limitPrecedesAggr = true; + } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java index 02aa64a..1b08880 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java @@ -764,8 +764,8 @@ public class CubeStorageQuery implements IStorageQuery { boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter) && context.isCoprocessorEnabled()); boolean goodSort = !context.hasSort(); if (goodAggr && goodFilter && goodSort) { - logger.info("Enable limit " + context.getLimit()); context.enableLimit(); + logger.info("Enable limit " + context.getLimit()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java index e8dd5b9..c4f7367 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java @@ -100,7 +100,7 @@ public class SerializedHBaseTupleIterator implements ITupleIterator { return true; // 1. check limit - if (context.isLimitEnabled() && scanCount >= context.getLimit() + context.getOffset()) { + if (context.isLimitEnabled() && (scanCount - context.getOffset() >= context.getLimit())) { return false; } // 2. check partial result http://git-wip-us.apache.org/repos/asf/kylin/blob/cdd945cb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java index da9c932..38efecc 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java @@ -254,14 +254,15 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement @Override public boolean hasNext() { + counter++; + if (counter > scanReq.getStorageScanRowNumThreshold()) { - throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter); + throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter + ", consider increasing kylin.query.memory-budget-bytes and kylin.query.scan-threshold"); } if (counter % (10 * GTScanRequest.terminateCheckInterval) == 1) { - logger.info("Scanned " + counter + " rows from HBase."); + logger.info("scanning " + counter + "th row from HBase."); } - counter++; return allCellLists.hasNext(); }
