KYLIN-1465 log improvement on others
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b9a3418b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b9a3418b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b9a3418b Branch: refs/heads/2.x-staging Commit: b9a3418b7f2121745090bb6463e7be754babfb73 Parents: b41c446 Author: Hongbin Ma <mahong...@apache.org> Authored: Fri Mar 4 15:59:37 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Fri Mar 4 16:01:04 2016 +0800 ---------------------------------------------------------------------- .../kylin/cube/gridtable/CubeGridTable.java | 6 ++--- .../gridtable/NotEnoughGTInfoException.java | 23 -------------------- .../kylin/gridtable/GTScanReqSerDerTest.java | 11 +++++----- .../kylin/storage/translate/HBaseKeyRange.java | 4 +--- .../streaming/cube/StreamingCubeBuilder.java | 6 +++-- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 14 ++++++------ .../storage/hbase/cube/v1/CubeStorageQuery.java | 6 ++--- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 18 +++++++-------- .../storage/hbase/cube/v2/CubeHBaseRPC.java | 8 +++---- .../hbase/cube/v2/CubeSegmentScanner.java | 3 +-- .../storage/hbase/cube/v2/CubeStorageQuery.java | 10 +++------ .../kylin/storage/hbase/cube/v2/RawScan.java | 2 +- 12 files changed, 41 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java index 05fc8a5..5f0bb07 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java @@ -29,18 +29,18 @@ public class CubeGridTable { dictionaryMap.put(col, dictionary); } } - + return dictionaryMap; } - public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) throws NotEnoughGTInfoException { + public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) { Map<TblColRef, Dictionary<String>> dictionaryMap = getDimensionToDictionaryMap(cubeSeg, cuboidId); Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId); for (TblColRef dim : cuboid.getColumns()) { if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(dim)) { Dictionary dict = dictionaryMap.get(dim); if (dict == null) { - throw new NotEnoughGTInfoException(); + throw new RuntimeException("Dictionary for " + dim + " is not found"); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java deleted file mode 100644 index 9bbcf75..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.gridtable; - -@SuppressWarnings("serial") -public class NotEnoughGTInfoException extends Exception { -} http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java index 6642d95..77cc2d8 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java @@ -29,7 +29,6 @@ import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.gridtable.CubeGridTable; -import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -73,7 +72,7 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase { buffer.flip(); GTInfo sInfo = GTInfo.serializer.deserialize(buffer); - this.compareTwoGTInfo(info,sInfo); + this.compareTwoGTInfo(info, sInfo); } @Test @@ -83,11 +82,11 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase { buffer.flip(); GTInfo sInfo = GTInfo.serializer.deserialize(buffer); - this.compareTwoGTInfo(info,sInfo); + this.compareTwoGTInfo(info, sInfo); } @Test - public void testGTInfo() throws NotEnoughGTInfoException { + public void testGTInfo() { CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready"); CubeSegment segment = cube.getFirstSegment(); @@ -95,8 +94,8 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase { GTInfo.serializer.serialize(info, buffer); buffer.flip(); - GTInfo sInfo = GTInfo.serializer.deserialize(buffer); - this.compareTwoGTInfo(info,sInfo); + GTInfo sInfo = GTInfo.serializer.deserialize(buffer); + this.compareTwoGTInfo(info, sInfo); } private void compareTwoGTInfo(GTInfo info, GTInfo sInfo) { http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java index a5ca62b..4f39cf6 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java @@ -45,9 +45,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; /** - * * @author xjiang - * */ public class HBaseKeyRange implements Comparable<HBaseKeyRange> { @@ -147,7 +145,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> { buf.append(BytesUtil.toHex(fuzzyKey.getFirst())); buf.append(" "); buf.append(BytesUtil.toHex(fuzzyKey.getSecond())); - buf.append(System.lineSeparator()); + buf.append(";"); } this.fuzzyKeyString = buf.toString(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java ---------------------------------------------------------------------- diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java index d7056cf..c4f2b7e 100644 --- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java +++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java @@ -73,6 +73,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder { private static final Logger logger = LoggerFactory.getLogger(StreamingCubeBuilder.class); private final String cubeName; + private int processedRowCount = 0; public StreamingCubeBuilder(String cubeName) { this.cubeName = cubeName; @@ -81,16 +82,16 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder { @Override public void build(StreamingBatch streamingBatch, Map<TblColRef, Dictionary<String>> dictionaryMap, ICuboidWriter cuboidWriter) { try { - CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName); LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue<List<String>>(); InMemCubeBuilder inMemCubeBuilder = new InMemCubeBuilder(cubeInstance.getDescriptor(), dictionaryMap); final Future<?> future = Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, cuboidWriter)); + processedRowCount = streamingBatch.getMessages().size(); for (StreamingMessage streamingMessage : streamingBatch.getMessages()) { blockingQueue.put(streamingMessage.getData()); } - blockingQueue.put(Collections.<String> emptyList()); + blockingQueue.put(Collections.<String>emptyList()); future.get(); cuboidWriter.flush(); @@ -157,6 +158,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder { public void commit(IBuildable buildable) { CubeSegment cubeSegment = (CubeSegment) buildable; cubeSegment.setStatus(SegmentStatusEnum.READY); + cubeSegment.setInputRecords(processedRowCount); CubeUpdate cubeBuilder = new CubeUpdate(cubeSegment.getCubeInstance()); cubeBuilder.setToAddSegs(cubeSegment); try { http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java index 5e842f7..909de39 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java @@ -210,26 +210,26 @@ public class CubeSegmentTupleIterator implements ITupleIterator { private void logScan(HBaseKeyRange keyRange) { StringBuilder info = new StringBuilder(); - info.append("\nScan hbase table ").append(tableName).append(": "); + info.append(" Scan hbase table ").append(tableName).append(": "); if (keyRange.getCuboid().requirePostAggregation()) { - info.append("cuboid require post aggregation, from "); + info.append(" cuboid require post aggregation, from "); } else { - info.append("cuboid exact match, from "); + info.append(" cuboid exact match, from "); } info.append(keyRange.getCuboid().getInputID()); info.append(" to "); info.append(keyRange.getCuboid().getId()); - info.append("\nStart: "); + info.append(" Start: "); info.append(keyRange.getStartKeyAsString()); info.append(" - "); info.append(Bytes.toStringBinary(keyRange.getStartKey())); - info.append("\nStop: "); + info.append(" Stop: "); info.append(keyRange.getStopKeyAsString()); info.append(" - "); info.append(Bytes.toStringBinary(keyRange.getStopKey())); if (this.scan.getFilter() != null) { - info.append("\nFuzzy key counts: " + keyRange.getFuzzyKeys().size()); - info.append("\nFuzzy: "); + info.append(" Fuzzy key counts: " + keyRange.getFuzzyKeys().size()); + info.append(" Fuzzy: "); info.append(keyRange.getFuzzyKeyAsString()); } logger.info(info.toString()); http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java index 1b8b586..3d7f620 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java @@ -493,7 +493,7 @@ public class CubeStorageQuery implements ICachableStorageQuery { List<Collection<ColumnValueRange>> result = Lists.newArrayList(); if (flatFilter == null) { - result.add(Collections.<ColumnValueRange> emptyList()); + result.add(Collections.<ColumnValueRange>emptyList()); return result; } @@ -535,7 +535,7 @@ public class CubeStorageQuery implements ICachableStorageQuery { } if (globalAlwaysTrue) { orAndRanges.clear(); - orAndRanges.add(Collections.<ColumnValueRange> emptyList()); + orAndRanges.add(Collections.<ColumnValueRange>emptyList()); } return orAndRanges; } @@ -762,7 +762,7 @@ public class CubeStorageQuery implements ICachableStorageQuery { private void setLimit(TupleFilter filter, StorageContext context) { boolean goodAggr = context.isExactAggregation(); boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter) && context.isCoprocessorEnabled()); - boolean goodSort = context.hasSort() == false; + boolean goodSort = !context.hasSort(); if (goodAggr && goodFilter && goodSort) { logger.info("Enable limit " + context.getLimit()); context.enableLimit(); http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index f22964f..e6f9ac1 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -293,15 +293,15 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { final AtomicInteger totalScannedCount = new AtomicInteger(0); final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(scanRequests.size() * shardNum); + final String currentThreadName = Thread.currentThread().getName(); for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) { executorService.submit(new Runnable() { @Override public void run() { for (int i = 0; i < scanRequests.size(); ++i) { - int scanIndex = i; CubeVisitProtos.CubeVisitRequest.Builder builder = CubeVisitProtos.CubeVisitRequest.newBuilder(); - builder.setGtScanRequest(scanRequestByteStrings.get(scanIndex)).setHbaseRawScan(rawScanByteStrings.get(scanIndex)); + builder.setGtScanRequest(scanRequestByteStrings.get(i)).setHbaseRawScan(rawScanByteStrings.get(i)); for (IntList intList : hbaseColumnsToGTIntList) { builder.addHbaseColumnsToGT(intList); } @@ -317,7 +317,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { for (Map.Entry<byte[], CubeVisitProtos.CubeVisitResponse> result : results.entrySet()) { totalScannedCount.addAndGet(result.getValue().getStats().getScannedRowCount()); - logger.info(getStatsString(result)); + logger.info("<spawned by " + currentThreadName + ">" + getStatsString(result)); try { epResultItr.append(CompressionUtils.decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows()))); } catch (IOException | DataFormatException e) { @@ -335,12 +335,12 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { private String getStatsString(Map.Entry<byte[], CubeVisitProtos.CubeVisitResponse> result) { StringBuilder sb = new StringBuilder(); Stats stats = result.getValue().getStats(); - sb.append("Endpoint RPC returned from HTable " + cubeSeg.getStorageLocationIdentifier() + " Shard " + BytesUtil.toHex(result.getKey()) + " on host: " + stats.getHostname() + "."); - sb.append("Total scanned row: " + stats.getScannedRowCount() + ". "); - sb.append("Total filtered/aggred row: " + stats.getAggregatedRowCount() + ". "); - sb.append("Time elapsed in EP: " + (stats.getServiceEndTime() - stats.getServiceStartTime()) + "(ms). "); - sb.append("Server CPU usage: " + stats.getSystemCpuLoad() + ", server physical mem left: " + stats.getFreePhysicalMemorySize() + ", server swap mem left:" + stats.getFreeSwapSpaceSize() + "."); - sb.append("Etc message: " + stats.getEtcMsg() + "."); + sb.append("Endpoint RPC returned from HTable ").append(cubeSeg.getStorageLocationIdentifier()).append(" Shard ").append(BytesUtil.toHex(result.getKey())).append(" on host: ").append(stats.getHostname()).append("."); + sb.append("Total scanned row: ").append(stats.getScannedRowCount()).append(". "); + sb.append("Total filtered/aggred row: ").append(stats.getAggregatedRowCount()).append(". "); + sb.append("Time elapsed in EP: ").append(stats.getServiceEndTime() - stats.getServiceStartTime()).append("(ms). "); + sb.append("Server CPU usage: ").append(stats.getSystemCpuLoad()).append(", server physical mem left: ").append(stats.getFreePhysicalMemorySize()).append(", server swap mem left:").append(stats.getFreeSwapSpaceSize()).append("."); + sb.append("Etc message: ").append(stats.getEtcMsg()).append("."); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java index db39455..eb5ac9f 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java @@ -273,20 +273,20 @@ public abstract class CubeHBaseRPC { info.append(cuboid.getInputID()); info.append(" to "); info.append(cuboid.getId()); - info.append("\nStart: "); + info.append(" Start: "); info.append(rawScan.getStartKeyAsString()); info.append(" ("); info.append(Bytes.toStringBinary(rawScan.startKey) + ")"); - info.append("\nStop: "); + info.append(" Stop: "); info.append(rawScan.getEndKeyAsString()); info.append(" ("); info.append(Bytes.toStringBinary(rawScan.endKey) + ")"); if (rawScan.fuzzyKeys != null && rawScan.fuzzyKeys.size() != 0) { - info.append("\nFuzzy key counts: " + rawScan.fuzzyKeys.size()); + info.append(" Fuzzy key counts: " + rawScan.fuzzyKeys.size()); info.append(". Fuzzy keys : "); info.append(rawScan.getFuzzyKeyAsString()); } else { - info.append("\nNo Fuzzy Key"); + info.append(", No Fuzzy Key"); } logger.info(info.toString()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java index 3f00566..ee5ae96 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java @@ -21,7 +21,6 @@ import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.gridtable.CubeGridTable; import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping; -import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.dict.TupleFilterFunctionTransformer; import org.apache.kylin.gridtable.EmptyGTScanner; @@ -56,7 +55,7 @@ public class CubeSegmentScanner implements IGTScanner { final Cuboid cuboid; public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, // - Collection<FunctionDesc> metrics, TupleFilter filter, boolean allowPreAggregate) throws NotEnoughGTInfoException { + Collection<FunctionDesc> metrics, TupleFilter filter, boolean allowPreAggregate) { this.cuboid = cuboid; this.cubeSeg = cubeSeg; this.info = CubeGridTable.newGTInfo(cubeSeg, cuboid.getId()); http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java index ab8c80f..df0bb84 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java @@ -11,7 +11,6 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.cuboid.Cuboid; -import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; import org.apache.kylin.dict.lookup.LookupStringTable; @@ -96,14 +95,11 @@ public class CubeStorageQuery implements ICachableStorageQuery { List<CubeSegmentScanner> scanners = Lists.newArrayList(); for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) { CubeSegmentScanner scanner; - try { - scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, !isExactAggregation); - } catch (NotEnoughGTInfoException e) { - //deal with empty cube segment - logger.info("Cannot construct Segment {}'s GTInfo, this may due to empty segment or broken metadata", cubeSeg); - logger.info("error stack", e); + if (cubeSeg.getInputRecords() == 0) { + logger.info("Skip cube segment {} because its input record is 0", cubeSeg); continue; } + scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, !isExactAggregation); scanners.add(scanner); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java index 361b1dd..c2ffdba 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java @@ -61,7 +61,7 @@ public class RawScan { buf.append(BytesUtil.toHex(fuzzyKey.getFirst())); buf.append(" "); buf.append(BytesUtil.toHex(fuzzyKey.getSecond())); - buf.append(System.lineSeparator()); + buf.append(";"); } return buf.toString(); }