This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push: new 985ff83 KYLIN-4896 Optimize the process of writing big resource files into HDFS. 985ff83 is described below commit 985ff834a2eefc536a3bb2e516ab36d8f5667893 Author: helenzeng0503 <linghui_helen_z...@163.com> AuthorDate: Fri Feb 5 09:40:30 2021 +0800 KYLIN-4896 Optimize the process of writing big resource files into HDFS. --- .../main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index fc00ec2..230ee73 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -88,8 +88,10 @@ public class HBaseResourceStore extends PushdownResourceStore { tableName = metadataUrl.getIdentifier(); createHTableIfNeeded(tableName); - kvSizeLimit = Integer + int kvSizeLimitActual = Integer .parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760")); + kvSizeLimit = kvSizeLimitActual > 10485760 ? kvSizeLimitActual : 10485760; + logger.debug("hbase.client.keyvalue.maxsize is {}, kvSizeLimit is set to {}", kvSizeLimitActual, kvSizeLimit); } protected Connection getConnection() throws IOException { @@ -328,6 +330,7 @@ public class HBaseResourceStore extends PushdownResourceStore { byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS); if (content.length > kvSizeLimit) { + logger.info("Length of content exceeds the limit of {} bytes, push down {} to HDFS", kvSizeLimit, resPath); pushdown = writePushdown(resPath, ContentWriter.create(content)); content = BytesUtil.EMPTY_BYTE_ARRAY; }