Repository: kylin Updated Branches: refs/heads/2.0-rc cbc86c265 -> b95d7282b
KYLIN-1205 Patch scan in HBaseResourceStore as well Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b95d7282 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b95d7282 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b95d7282 Branch: refs/heads/2.0-rc Commit: b95d7282babbf13b6d1b0f22760519db43e7a2a0 Parents: cbc86c2 Author: Yang Li <liy...@apache.org> Authored: Fri Dec 25 20:52:48 2015 +0800 Committer: Yang Li <liy...@apache.org> Committed: Fri Dec 25 20:52:48 2015 +0800 ---------------------------------------------------------------------- .../apache/kylin/storage/hbase/HBaseResourceStore.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b95d7282/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index 6f638c4..511a7cf 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -156,6 +156,7 @@ public class HBaseResourceStore extends ResourceStore { Scan scan = new Scan(startRow, endRow); scan.addColumn(B_FAMILY, B_COLUMN_TS); scan.addColumn(B_FAMILY, B_COLUMN); + tuneScanParameters(scan); HTableInterface table = getConnection().getTable(getAllInOneTableName()); List<RawResource> result = Lists.newArrayList(); @@ -175,6 +176,13 @@ public class HBaseResourceStore extends ResourceStore { return result; } + private void tuneScanParameters(Scan scan) { + // divide by 10 as some resource like dictionary or snapshot can be very large + scan.setCaching(kylinConfig.getHBaseScanCacheRows() / 10); + scan.setMaxResultSize(kylinConfig.getHBaseScanMaxResultSize()); + scan.setCacheBlocks(true); + } + private InputStream getInputStream(String resPath, Result r) throws IOException { if (r == null) { return null; @@ -212,7 +220,7 @@ public class HBaseResourceStore extends ResourceStore { protected long getResourceTimestampImpl(String resPath) throws IOException { return getTimestamp(getByScan(resPath, false, true)); } - + @Override protected void putResourceImpl(String resPath, InputStream content, long ts) throws IOException { ByteArrayOutputStream bout = new ByteArrayOutputStream();