KYLIN-1565: Read KV max size from HBase config
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e05cb600 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e05cb600 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e05cb600 Branch: refs/heads/orderedbytes Commit: e05cb60061a43987f7648d3d04921b5a01bb5ac9 Parents: 0861bfb Author: Yiming Liu <liuyiming....@gmail.com> Authored: Fri Sep 16 11:51:11 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Mon Sep 26 19:45:19 2016 +0800 ---------------------------------------------------------------------- .../java/org/apache/kylin/common/KylinConfigBase.java | 4 ---- .../kylin/storage/hbase/ITHBaseResourceStoreTest.java | 10 +++++----- .../apache/kylin/storage/hbase/HBaseResourceStore.java | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e05cb600/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 79ee084..ceb188e 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -570,10 +570,6 @@ abstract public class KylinConfigBase implements Serializable { return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft); } - public int getHBaseKeyValueSize() { - return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760")); - } - public String getDefaultIGTStorage() { return getOptional("kylin.query.storage.default.gtstorage", "org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC"); } http://git-wip-us.apache.org/repos/asf/kylin/blob/e05cb600/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java index bc5cdf1..5a2d5dd 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java @@ -57,12 +57,12 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase { String path = "/cube/_test_large_cell.json"; String largeContent = "THIS_IS_A_LARGE_CELL"; StringEntity content = new StringEntity(largeContent); - KylinConfig config = KylinConfig.getInstanceFromEnv(); - int origSize = config.getHBaseKeyValueSize(); + Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); + int origSize = Integer.parseInt(hconf.get("hbase.client.keyvalue.maxsize", "10485760")); ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); try { - config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1)); + hconf.set("hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1)); store.deleteResource(path); @@ -72,7 +72,7 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase { assertEquals(content, t); Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path); - Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); + FileSystem fileSystem = FileSystem.get(hconf); assertTrue(fileSystem.exists(redirectPath)); @@ -82,7 +82,7 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase { store.deleteResource(path); } finally { - config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize); + hconf.set("hbase.client.keyvalue.maxsize", "" + origSize); store.deleteResource(path); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/e05cb600/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index e2f3661..ee4b7e4 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -364,7 +364,7 @@ public class HBaseResourceStore extends ResourceStore { } private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException { - int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize(); + int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760")); if (content.length > kvSizeLimit) { writeLargeCellToHdfs(resPath, content, table); content = BytesUtil.EMPTY_BYTE_ARRAY;
