Repository: kylin Updated Branches: refs/heads/master 5860c497e -> da53a7716
KYLIN-1936 limit push down only applys to ID_SHARDED_HBASE Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da53a771 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da53a771 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da53a771 Branch: refs/heads/master Commit: da53a7716ae42dc831f16411f04097ef743ef47c Parents: 5860c49 Author: Hongbin Ma <mahong...@apache.org> Authored: Tue Aug 16 18:13:10 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Aug 16 18:13:18 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/model/CubeDesc.java | 9 +++++-- .../storage/gtrecord/CubeSegmentScanner.java | 4 ++++ .../gtrecord/FetchSourceAwareIterator.java | 24 ------------------- .../storage/gtrecord/IFetchSourceAware.java | 25 -------------------- .../gtrecord/SequentialCubeTupleIterator.java | 6 +++-- .../apache/kylin/query/ITKylinQueryTest.java | 5 +--- 6 files changed, 16 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 6186707..c0f3ed8 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -29,9 +29,9 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import java.util.Map.Entry; import javax.annotation.Nullable; @@ -64,9 +64,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Function; import com.google.common.collect.Collections2; @@ -994,6 +994,11 @@ public class CubeDesc extends RootPersistentEntity { this.nullStrings = nullStrings; } + public boolean supportsLimitPushDown() { + //currently only ID_SHARDED_HBASE supports limit push down + return getStorageType() == IStorageAware.ID_SHARDED_HBASE; + } + public int getStorageType() { return storageType; } http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java index 83ee6c7..49dc5cf 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java @@ -97,4 +97,8 @@ public class CubeSegmentScanner implements IGTScanner { return scanner.getScannedRowCount(); } + public CubeSegment getSegment() { + return this.cubeSeg; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/FetchSourceAwareIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/FetchSourceAwareIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/FetchSourceAwareIterator.java deleted file mode 100644 index cb83819..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/FetchSourceAwareIterator.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.gtrecord; - -import java.util.Iterator; - -interface FetchSourceAwareIterator<F> extends IFetchSourceAware<F>, Iterator<F> { -} http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IFetchSourceAware.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IFetchSourceAware.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IFetchSourceAware.java deleted file mode 100644 index d51ca4c..0000000 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IFetchSourceAware.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.storage.gtrecord; - -import java.util.Iterator; - -public interface IFetchSourceAware<E> { - public Iterator<? extends E> getFetchSource(); -} http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java index ff7fb2b..2cff76c 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java @@ -62,9 +62,11 @@ public class SequentialCubeTupleIterator implements ITupleIterator { for (CubeSegmentScanner scanner : scanners) { segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context)); } - + + boolean supportLimitPushDown = scanners.get(0).getSegment().getCubeDesc().supportsLimitPushDown(); + this.storagePushDownLimit = context.getStoragePushDownLimit(); - if (storagePushDownLimit > KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax()) { + if (!supportLimitPushDown || storagePushDownLimit > KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax()) { //normal case tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator()); } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index 5db4342..741dd18 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -35,7 +35,6 @@ import org.apache.kylin.query.enumerator.OLAPQuery; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.routing.Candidate; import org.apache.kylin.query.schema.OLAPSchemaFactory; -import org.apache.kylin.storage.hbase.HBaseStorage; import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler; import org.dbunit.database.DatabaseConnection; import org.dbunit.database.IDatabaseConnection; @@ -261,9 +260,7 @@ public class ITKylinQueryTest extends KylinTestBase { @Test public void testLimitCorrectness() throws Exception { - if (HBaseStorage.overwriteStorageQuery == null) {//v1 query engine will not work - execLimitAndValidate(getQueryFolderPrefix() + "src/test/resources/query/sql"); - } + execLimitAndValidate(getQueryFolderPrefix() + "src/test/resources/query/sql"); } @Test