Repository: hbase Updated Branches: refs/heads/branch-1.2 65738a835 -> f8762efc0
HBASE-16032 Possible memory leak in StoreScanner Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f8762efc Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f8762efc Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f8762efc Branch: refs/heads/branch-1.2 Commit: f8762efc0e041ba63c745bfa3ccb9f06d2010699 Parents: 65738a8 Author: Yu Li <[email protected]> Authored: Fri Jun 17 18:00:16 2016 +0800 Committer: Yu Li <[email protected]> Committed: Wed Jun 22 02:06:03 2016 +0800 ---------------------------------------------------------------------- .../hadoop/hbase/regionserver/HRegion.java | 41 ++++++++++++------- .../hadoop/hbase/regionserver/StoreScanner.java | 43 ++++++++++++-------- 2 files changed, 52 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/f8762efc/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index c4b4f5a..04f0b87 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -5523,26 +5523,39 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi // by the filter to operate (scanners list) and all others (joinedScanners list). List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>(); List<KeyValueScanner> joinedScanners = new ArrayList<KeyValueScanner>(); - if (additionalScanners != null) { + // Store all already instantiated scanners for exception handling + List<KeyValueScanner> instantiatedScanners = new ArrayList<KeyValueScanner>(); + // handle additionalScanners + if (additionalScanners != null && !additionalScanners.isEmpty()) { scanners.addAll(additionalScanners); + instantiatedScanners.addAll(additionalScanners); } - for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) { - Store store = stores.get(entry.getKey()); - KeyValueScanner scanner; - try { - scanner = store.getScanner(scan, entry.getValue(), this.readPt); - } catch (FileNotFoundException e) { - throw handleFileNotFound(e); + try { + for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) { + Store store = stores.get(entry.getKey()); + KeyValueScanner scanner; + try { + scanner = store.getScanner(scan, entry.getValue(), this.readPt); + } catch (FileNotFoundException e) { + throw handleFileNotFound(e); + } + instantiatedScanners.add(scanner); + if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand() + || this.filter.isFamilyEssential(entry.getKey())) { + scanners.add(scanner); + } else { + joinedScanners.add(scanner); + } } - if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand() - || this.filter.isFamilyEssential(entry.getKey())) { - scanners.add(scanner); - } else { - joinedScanners.add(scanner); + initializeKVHeap(scanners, joinedScanners, region); + } catch (IOException e) { + // close all already instantiated scanners before throwing the exception + for (KeyValueScanner scanner : instantiatedScanners) { + scanner.close(); } + throw e; } - initializeKVHeap(scanners, joinedScanners, region); } protected void initializeKVHeap(List<KeyValueScanner> scanners, http://git-wip-us.apache.org/repos/asf/hbase/blob/f8762efc/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index a026a40..30d634e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -188,24 +188,31 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner this.store.addChangedReaderObserver(this); - // Pass columns to try to filter out unnecessary StoreFiles. - List<KeyValueScanner> scanners = getScannersNoCompaction(); - - // Seek all scanners to the start of the Row (or if the exact matching row - // key does not exist, then to the start of the next matching Row). - // Always check bloom filter to optimize the top row seek for delete - // family marker. - seekScanners(scanners, matcher.getStartKey(), explicitColumnQuery - && lazySeekEnabledGlobally, parallelSeekEnabled); - - // set storeLimit - this.storeLimit = scan.getMaxResultsPerColumnFamily(); - - // set rowOffset - this.storeOffset = scan.getRowOffsetPerColumnFamily(); - - // Combine all seeked scanners with a heap - resetKVHeap(scanners, store.getComparator()); + try { + // Pass columns to try to filter out unnecessary StoreFiles. + List<KeyValueScanner> scanners = getScannersNoCompaction(); + + // Seek all scanners to the start of the Row (or if the exact matching row + // key does not exist, then to the start of the next matching Row). + // Always check bloom filter to optimize the top row seek for delete + // family marker. + seekScanners(scanners, matcher.getStartKey(), explicitColumnQuery && lazySeekEnabledGlobally, + parallelSeekEnabled); + + // set storeLimit + this.storeLimit = scan.getMaxResultsPerColumnFamily(); + + // set rowOffset + this.storeOffset = scan.getRowOffsetPerColumnFamily(); + + // Combine all seeked scanners with a heap + resetKVHeap(scanners, store.getComparator()); + } catch (IOException e) { + // remove us from the HStore#changedReaderObservers here or we'll have no chance to + // and might cause memory leak + this.store.deleteChangedReaderObserver(this); + throw e; + } } /**
