Repository: hbase Updated Branches: refs/heads/branch-1.3 c8501d800 -> 0cbad9c89
HBASE-16032 Possible memory leak in StoreScanner Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0cbad9c8 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0cbad9c8 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0cbad9c8 Branch: refs/heads/branch-1.3 Commit: 0cbad9c890b65b1409be741f55370b7a387f5764 Parents: c8501d8 Author: Yu Li <[email protected]> Authored: Fri Jun 17 18:00:16 2016 +0800 Committer: Yu Li <[email protected]> Committed: Wed Jun 22 01:54:44 2016 +0800 ---------------------------------------------------------------------- .../hadoop/hbase/regionserver/HRegion.java | 41 ++++++++++++------- .../hadoop/hbase/regionserver/StoreScanner.java | 43 ++++++++++++-------- 2 files changed, 52 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/0cbad9c8/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index b41502e..09c118b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -5685,26 +5685,39 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>(scan.getFamilyMap().size()); List<KeyValueScanner> joinedScanners = new ArrayList<KeyValueScanner>(scan.getFamilyMap().size()); - if (additionalScanners != null) { + // Store all already instantiated scanners for exception handling + List<KeyValueScanner> instantiatedScanners = new ArrayList<KeyValueScanner>(); + // handle additionalScanners + if (additionalScanners != null && !additionalScanners.isEmpty()) { scanners.addAll(additionalScanners); + instantiatedScanners.addAll(additionalScanners); } - for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) { - Store store = stores.get(entry.getKey()); - KeyValueScanner scanner; - try { - scanner = store.getScanner(scan, entry.getValue(), this.readPt); - } catch (FileNotFoundException e) { - throw handleFileNotFound(e); + try { + for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) { + Store store = stores.get(entry.getKey()); + KeyValueScanner scanner; + try { + scanner = store.getScanner(scan, entry.getValue(), this.readPt); + } catch (FileNotFoundException e) { + throw handleFileNotFound(e); + } + instantiatedScanners.add(scanner); + if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand() + || this.filter.isFamilyEssential(entry.getKey())) { + scanners.add(scanner); + } else { + joinedScanners.add(scanner); + } } - if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand() - || this.filter.isFamilyEssential(entry.getKey())) { - scanners.add(scanner); - } else { - joinedScanners.add(scanner); + initializeKVHeap(scanners, joinedScanners, region); + } catch (IOException e) { + // close all already instantiated scanners before throwing the exception + for (KeyValueScanner scanner : instantiatedScanners) { + scanner.close(); } + throw e; } - initializeKVHeap(scanners, joinedScanners, region); } protected void initializeKVHeap(List<KeyValueScanner> scanners, http://git-wip-us.apache.org/repos/asf/hbase/blob/0cbad9c8/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index fe8a51b..8eaa693 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -198,24 +198,31 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner this.store.addChangedReaderObserver(this); - // Pass columns to try to filter out unnecessary StoreFiles. - List<KeyValueScanner> scanners = getScannersNoCompaction(); - - // Seek all scanners to the start of the Row (or if the exact matching row - // key does not exist, then to the start of the next matching Row). - // Always check bloom filter to optimize the top row seek for delete - // family marker. - seekScanners(scanners, matcher.getStartKey(), explicitColumnQuery - && lazySeekEnabledGlobally, parallelSeekEnabled); - - // set storeLimit - this.storeLimit = scan.getMaxResultsPerColumnFamily(); - - // set rowOffset - this.storeOffset = scan.getRowOffsetPerColumnFamily(); - addCurrentScanners(scanners); - // Combine all seeked scanners with a heap - resetKVHeap(scanners, store.getComparator()); + try { + // Pass columns to try to filter out unnecessary StoreFiles. + List<KeyValueScanner> scanners = getScannersNoCompaction(); + + // Seek all scanners to the start of the Row (or if the exact matching row + // key does not exist, then to the start of the next matching Row). + // Always check bloom filter to optimize the top row seek for delete + // family marker. + seekScanners(scanners, matcher.getStartKey(), explicitColumnQuery && lazySeekEnabledGlobally, + parallelSeekEnabled); + + // set storeLimit + this.storeLimit = scan.getMaxResultsPerColumnFamily(); + + // set rowOffset + this.storeOffset = scan.getRowOffsetPerColumnFamily(); + addCurrentScanners(scanners); + // Combine all seeked scanners with a heap + resetKVHeap(scanners, store.getComparator()); + } catch (IOException e) { + // remove us from the HStore#changedReaderObservers here or we'll have no chance to + // and might cause memory leak + this.store.deleteChangedReaderObserver(this); + throw e; + } } /**
