refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a7a318c2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a7a318c2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a7a318c2 Branch: refs/heads/mhb0220 Commit: a7a318c214a8b03d8a133b7ec035765fceb5f734 Parents: 661f016 Author: Hongbin Ma <mahong...@apache.org> Authored: Mon Feb 20 10:15:53 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Feb 21 20:55:36 2017 +0800 ---------------------------------------------------------------------- .../cube/inmemcubing/ConcurrentDiskStore.java | 7 +---- .../cube/inmemcubing/InMemCubeBuilder.java | 5 --- .../kylin/cube/inmemcubing/MemDiskStore.java | 6 ---- .../apache/kylin/gridtable/EmptyGTScanner.java | 9 +----- .../kylin/gridtable/GTAggregateScanner.java | 7 +---- .../apache/kylin/gridtable/GTFilterScanner.java | 5 --- .../apache/kylin/gridtable/GTScanRequest.java | 9 +++--- .../org/apache/kylin/gridtable/IGTScanner.java | 4 --- .../benchmark/SortedGTRecordGenerator.java | 6 ---- .../gridtable/memstore/GTSimpleMemStore.java | 5 --- .../gridtable/AggregationCacheSpillTest.java | 12 +------ .../kylin/gridtable/SimpleGridTableTest.java | 9 +----- .../storage/gtrecord/CubeSegmentScanner.java | 5 --- .../kylin/storage/gtrecord/ScannerWorker.java | 15 +++------ .../gtrecord/StorageResponseGTScatter.java | 9 +----- .../hbase/cube/HBaseScannerBenchmark.java | 3 -- .../storage/hbase/cube/SimpleHBaseStore.java | 5 --- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 22 ++++++------- .../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 33 ++------------------ .../hbase/cube/v2/HBaseReadonlyStore.java | 5 --- .../endpoint/protobuf/CubeVisit.proto | 2 +- .../hbase/steps/SandboxMetastoreCLI.java | 1 - 22 files changed, 27 insertions(+), 157 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java index 5219ede..41d2dfb 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java @@ -265,11 +265,6 @@ public class ConcurrentDiskStore implements IGTStore, Closeable { return info; } - @Override - public long getScannedRowCount() { - return count; - } - } private class Writer implements IGTWriter { @@ -371,4 +366,4 @@ public class ConcurrentDiskStore implements IGTStore, Closeable { return "ConcurrentDiskStore@" + (info.getTableName() == null ? this.hashCode() : info.getTableName()); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java index a74f0c0..e08844e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java @@ -564,10 +564,5 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { public GTInfo getInfo() { return info; } - - @Override - public long getScannedRowCount() { - return 0L; - } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java index 81403ab..a5471df 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java @@ -277,12 +277,6 @@ public class MemDiskStore implements IGTStore, Closeable { public GTInfo getInfo() { return info; } - - @Override - public long getScannedRowCount() { - return count; - } - } private class Writer implements IGTWriter { http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java index 01d31f0..8b6c995 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java @@ -22,10 +22,8 @@ import java.io.IOException; import java.util.Iterator; public class EmptyGTScanner implements IGTScanner { - private long reportScannedRowCount; - public EmptyGTScanner(long reportScannedRowCount) { - this.reportScannedRowCount = reportScannedRowCount; + public EmptyGTScanner() { } @Override @@ -34,11 +32,6 @@ public class EmptyGTScanner implements IGTScanner { } @Override - public long getScannedRowCount() { - return reportScannedRowCount; - } - - @Override public void close() throws IOException { } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java index 8b0efcc..7cdd4f5 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java @@ -129,11 +129,6 @@ public class GTAggregateScanner implements IGTScanner { } @Override - public long getScannedRowCount() { - return inputScanner.getScannedRowCount(); - } - - @Override public void close() throws IOException { inputScanner.close(); aggrCache.close(); @@ -598,4 +593,4 @@ public class GTAggregateScanner implements IGTScanner { } } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java index f1f84af..717f89c 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java @@ -63,11 +63,6 @@ public class GTFilterScanner implements IGTScanner { } @Override - public long getScannedRowCount() { - return inputScanner.getScannedRowCount(); - } - - @Override public void close() throws IOException { inputScanner.close(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index 651e5c4..4629c8e 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -42,7 +42,7 @@ import com.google.common.collect.Sets; public class GTScanRequest { private static final Logger logger = LoggerFactory.getLogger(GTScanRequest.class); - + //it's not necessary to increase the checkInterval to very large because the check cost is not high //changing it might break org.apache.kylin.query.ITKylinQueryTest.testTimeoutQuery() public static final int terminateCheckInterval = 100; @@ -175,8 +175,8 @@ public class GTScanRequest { public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, boolean hasPreFiltered, boolean spillEnabled) throws IOException { IGTScanner result = scanner; if (!filterToggledOn) { //Skip reading this section if you're not profiling! - int scanned = lookAndForget(result); - return new EmptyGTScanner(scanned); + lookAndForget(result); + return new EmptyGTScanner(); } else { if (this.hasFilterPushDown() && !hasPreFiltered) { @@ -184,9 +184,8 @@ public class GTScanRequest { } if (!aggrToggledOn) {//Skip reading this section if you're not profiling! - long scanned = result.getScannedRowCount(); lookAndForget(result); - return new EmptyGTScanner(scanned); + return new EmptyGTScanner(); } if (!this.isAllowStorageAggregation()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java index 980787b..96c7972 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java @@ -21,9 +21,5 @@ package org.apache.kylin.gridtable; import java.io.Closeable; public interface IGTScanner extends Iterable<GTRecord>, Closeable { - GTInfo getInfo(); - - long getScannedRowCount(); - } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java index 9c839bb..71a0a21 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java @@ -184,12 +184,6 @@ public class SortedGTRecordGenerator { public GTInfo getInfo() { return info; } - - @Override - public long getScannedRowCount() { - return counter; - } - } private class Distribution { http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java index f9f370b..e1b5406 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java @@ -106,11 +106,6 @@ public class GTSimpleMemStore implements IGTStore { } @Override - public long getScannedRowCount() { - return count; - } - - @Override public void close() throws IOException { } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java index 7abb069..8b2243c 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java @@ -68,11 +68,6 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { } @Override - public long getScannedRowCount() { - throw new UnsupportedOperationException(); - } - - @Override public void close() throws IOException { } @@ -109,11 +104,6 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { } @Override - public long getScannedRowCount() { - throw new UnsupportedOperationException(); - } - - @Override public void close() throws IOException { } @@ -141,4 +131,4 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { assertEquals(10, count); scanner.close(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java index 4ac6644..14a25c5 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java @@ -18,7 +18,6 @@ package org.apache.kylin.gridtable; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -53,7 +52,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { GTBuilder builder = rebuild(table); IGTScanner scanner = scan(table); - assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount()); } @Test @@ -64,7 +62,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { GTBuilder builder = rebuild(table); IGTScanner scanner = scan(table); - assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount()); } @Test @@ -75,7 +72,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { GTBuilder builder = rebuild(table); IGTScanner scanner = scanAndAggregate(table); - assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount()); } @Test @@ -86,7 +82,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { rebuildViaAppend(table); IGTScanner scanner = scan(table); - assertEquals(10, scanner.getScannedRowCount()); } private IGTScanner scan(GridTable table) throws IOException { @@ -101,12 +96,11 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { System.out.println(r); } scanner.close(); - System.out.println("Scanned Row Count: " + scanner.getScannedRowCount()); return scanner; } private IGTScanner scanAndAggregate(GridTable table) throws IOException { - GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4)).setAggrMetricsFuncs(new String[]{"count", "sum"}).setFilterPushDown(null).createGTScanRequest(); + GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4)).setAggrMetricsFuncs(new String[] { "count", "sum" }).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = table.scan(req); int i = 0; for (GTRecord r : scanner) { @@ -135,7 +129,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { System.out.println(r); } scanner.close(); - System.out.println("Scanned Row Count: " + scanner.getScannedRowCount()); return scanner; } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java index 029502c..4f206d4 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java @@ -96,11 +96,6 @@ public class CubeSegmentScanner implements IGTScanner { return scanRequest == null ? null : scanRequest.getInfo(); } - @Override - public long getScannedRowCount() { - return scanner.getScannedRowCount(); - } - public CubeSegment getSegment() { return this.cubeSeg; } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java index fd50c54..9e89227 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java @@ -18,6 +18,10 @@ package org.apache.kylin.storage.gtrecord; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.Iterator; + import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.gridtable.EmptyGTScanner; import org.apache.kylin.gridtable.GTInfo; @@ -29,10 +33,6 @@ import org.apache.kylin.metadata.model.ISegment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.util.Iterator; - public class ScannerWorker { private static final Logger logger = LoggerFactory.getLogger(ScannerWorker.class); @@ -41,7 +41,7 @@ public class ScannerWorker { public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) { if (scanRequest == null) { logger.info("Segment {} will be skipped", segment); - internal = new EmptyGTScanner(0); + internal = new EmptyGTScanner(); return; } @@ -62,9 +62,4 @@ public class ScannerWorker { public void close() throws IOException { internal.close(); } - - public long getScannedRowCount() { - return internal.getScannedRowCount(); - } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java index dc8746f..3904b5c 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java @@ -45,15 +45,13 @@ public class StorageResponseGTScatter implements IGTScanner { private IPartitionStreamer partitionStreamer; private Iterator<byte[]> blocks; private ImmutableBitSet columns; - private long totalScannedCount; private int storagePushDownLimit = -1; - public StorageResponseGTScatter(GTInfo info, IPartitionStreamer partitionStreamer, ImmutableBitSet columns, long totalScannedCount, int storagePushDownLimit) { + public StorageResponseGTScatter(GTInfo info, IPartitionStreamer partitionStreamer, ImmutableBitSet columns, int storagePushDownLimit) { this.info = info; this.partitionStreamer = partitionStreamer; this.blocks = partitionStreamer.asByteArrayIterator(); this.columns = columns; - this.totalScannedCount = totalScannedCount; this.storagePushDownLimit = storagePushDownLimit; } @@ -63,11 +61,6 @@ public class StorageResponseGTScatter implements IGTScanner { } @Override - public long getScannedRowCount() { - return totalScannedCount; - } - - @Override public void close() throws IOException { //If upper consumer failed while consuming the GTRecords, the consumer should call IGTScanner's close method to ensure releasing resource partitionStreamer.close(); http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java index 3fdb92f..3eecba1 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java @@ -132,9 +132,6 @@ public class HBaseScannerBenchmark { } scan.close(); - if (scan.getScannedRowCount() != count) - throw new IllegalStateException(); - t = System.currentTimeMillis() - t; logger.info(msg + ", " + count + " records, " + speed(t) + "K rec/sec"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java index f63d9c2..b12173d 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java @@ -205,10 +205,5 @@ public class SimpleHBaseStore implements IGTStore { public GTInfo getInfo() { return info; } - - @Override - public long getScannedRowCount() { - return count; - } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index abc3437..82b67b6 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -23,7 +23,6 @@ import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.zip.DataFormatException; @@ -146,8 +145,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { logger.debug("Submitting rpc to {} shards starting from shard {}, scan range count {}", shardNum, cuboidBaseShard, rawScans.size()); - final AtomicLong totalScannedCount = new AtomicLong(0); - // KylinConfig: use env instance instead of CubeSegment, because KylinConfig will share among queries // for different cubes until redeployment of coprocessor jar. final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); @@ -205,7 +202,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { Stats stats = result.getStats(); queryContext.addAndGetScannedRows(stats.getScannedRowCount()); queryContext.addAndGetScannedBytes(stats.getScannedBytes()); - totalScannedCount.addAndGet(stats.getScannedRowCount()); // if any other region has responded with error, skip further processing if (regionErrorHolder.get() != null) { @@ -249,7 +245,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { }); } - return new StorageResponseGTScatter(fullGTInfo, new DummyPartitionStreamer(epResultItr), scanRequest.getColumns(), totalScannedCount.get(), scanRequest.getStoragePushDownLimit()); + return new StorageResponseGTScatter(fullGTInfo, new DummyPartitionStreamer(epResultItr), scanRequest.getColumns(), scanRequest.getStoragePushDownLimit()); } private ByteString serializeGTScanReq(GTScanRequest scanRequest) { @@ -317,14 +313,14 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { CubeVisitResponse.ErrorInfo errorInfo = response.getErrorInfo(); switch (errorInfo.getType()) { - case UNKNOWN_TYPE: - return new RuntimeException("Coprocessor aborts: " + errorInfo.getMessage()); - case TIMEOUT: - return new KylinTimeoutException(errorInfo.getMessage()); - case RESOURCE_LIMIT_EXCEEDED: - return new ResourceLimitExceededException("Coprocessor resource limit exceeded: " + errorInfo.getMessage()); - default: - throw new AssertionError("Unknown error type: " + errorInfo.getType()); + case UNKNOWN_TYPE: + return new RuntimeException("Coprocessor aborts: " + errorInfo.getMessage()); + case TIMEOUT: + return new KylinTimeoutException(errorInfo.getMessage()); + case RESOURCE_LIMIT_EXCEEDED: + return new ResourceLimitExceededException("Coprocessor resource limit exceeded: " + errorInfo.getMessage()); + default: + throw new AssertionError("Unknown error type: " + errorInfo.getType()); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java index 1698180..33f8d90 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java @@ -93,31 +93,7 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC { @Override public IGTScanner getGTScanner(final GTScanRequest scanRequest) throws IOException { - final IGTScanner scanner = getGTScannerInternal(scanRequest); - - return new IGTScanner() { - @Override - public GTInfo getInfo() { - return scanner.getInfo(); - } - - @Override - public long getScannedRowCount() { - long sum = 0; - sum += scanner.getScannedRowCount(); - return sum; - } - - @Override - public void close() throws IOException { - scanner.close(); - } - - @Override - public Iterator<GTRecord> iterator() { - return scanner.iterator(); - } - }; + return getGTScannerInternal(scanRequest); } //for non-sharding cases it will only return one byte[] with not shard at beginning @@ -229,11 +205,6 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC { } @Override - public long getScannedRowCount() { - return decorateScanner.getScannedRowCount(); - } - - @Override public void close() throws IOException { decorateScanner.close(); } @@ -244,4 +215,4 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC { } }; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java index 4c02dff..631e8e8 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java @@ -152,11 +152,6 @@ public class HBaseReadonlyStore implements IGTStore { public GTInfo getInfo() { return info; } - - @Override - public long getScannedRowCount() { - return count; - } }; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto index e01ff52..aa83595 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto @@ -37,7 +37,7 @@ message CubeVisitRequest { required string kylinProperties = 5; // kylin properties optional string queryId = 6; optional bool spillEnabled = 7 [default = true]; - optional int64 maxScanBytes = 8; // 0 means no limit + optional int64 maxScanBytes = 8; // must be positive message IntList { repeated int32 ints = 1; } http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java index 691886b..62b154e 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java @@ -36,7 +36,6 @@ import org.slf4j.LoggerFactory; * It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment */ public class SandboxMetastoreCLI { - private static final Logger logger = LoggerFactory.getLogger(SandboxMetastoreCLI.class); public static void main(String[] args) throws Exception {