KYLIN-1740: HDFS storage leak in HBaseResourceStore.deleteResourceImpl() in case of large cell of dictionary or table snapshot
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76d60b66 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76d60b66 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76d60b66 Branch: refs/heads/stream_m1 Commit: 76d60b66c59dfea78734e33fac9c5aa3535bff34 Parents: 762992d Author: Ma Gang <mg4w...@163.com> Authored: Mon Jun 20 12:20:36 2016 +0800 Committer: Zhong <yangzh...@lm-shc-16501214.corp.ebay.com> Committed: Mon Jun 20 12:22:21 2016 +0800 ---------------------------------------------------------------------- .../kylin/storage/hbase/HBaseResourceStore.java | 39 ++++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/76d60b66/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index 2262482..833c31c 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -277,9 +277,28 @@ public class HBaseResourceStore extends ResourceStore { protected void deleteResourceImpl(String resPath) throws IOException { HTableInterface table = getConnection().getTable(getAllInOneTableName()); try { + boolean hdfsResourceExist = false; + Result result = internalGetFromHTable(table,resPath,true,false); + if (result != null) { + byte[] value = result.getValue(B_FAMILY, B_COLUMN); + if (value != null && value.length == 0) { + hdfsResourceExist = true; + } + } + Delete del = new Delete(Bytes.toBytes(resPath)); table.delete(del); table.flushCommits(); + + if (hdfsResourceExist) { // remove hdfs cell value + Path redirectPath = bigCellHDFSPath(resPath); + Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); + FileSystem fileSystem = FileSystem.get(hconf); + + if (fileSystem.exists(redirectPath)) { + fileSystem.delete(redirectPath, true); + } + } } finally { IOUtils.closeQuietly(table); } @@ -291,6 +310,15 @@ public class HBaseResourceStore extends ResourceStore { } private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException { + HTableInterface table = getConnection().getTable(getAllInOneTableName()); + try { + return internalGetFromHTable(table,path,fetchContent,fetchTimestamp); + } finally { + IOUtils.closeQuietly(table); + } + } + + private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException { byte[] rowkey = Bytes.toBytes(path); Get get = new Get(rowkey); @@ -304,14 +332,9 @@ public class HBaseResourceStore extends ResourceStore { get.addColumn(B_FAMILY, B_COLUMN_TS); } - HTableInterface table = getConnection().getTable(getAllInOneTableName()); - try { - Result result = table.get(get); - boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists())); - return exists ? result : null; - } finally { - IOUtils.closeQuietly(table); - } + Result result = table.get(get); + boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists())); + return exists ? result : null; } private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {