Repository: hbase Updated Branches: refs/heads/branch-1 b0780bdc6 -> 4b8195f22
HBASE-17057 Minor compactions should also drop page cache (Ashu Pachauri) Signed-off-by: Gary Helmling <ga...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b8195f2 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b8195f2 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b8195f2 Branch: refs/heads/branch-1 Commit: 4b8195f22c1adf414d96f7e9f819c95ea6333479 Parents: b0780bd Author: Ashu Pachauri <ashu210...@gmail.com> Authored: Wed Nov 30 15:47:59 2016 -0800 Committer: Gary Helmling <ga...@apache.org> Committed: Fri Feb 24 15:31:36 2017 -0800 ---------------------------------------------------------------------- .../src/main/resources/hbase-default.xml | 18 ++++++++++++++ .../regionserver/compactions/Compactor.java | 26 ++++++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/4b8195f2/hbase-common/src/main/resources/hbase-default.xml ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 147e243..7cd0015 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -753,6 +753,24 @@ possible configurations would overwhelm and obscure the important. </description> </property> <property> + <name>hbase.regionserver.majorcompaction.pagecache.drop</name> + <value>true</value> + <description>Specifies whether to drop pages read/written into the system page cache by + major compactions. Setting it to true helps prevent major compactions from + polluting the page cache, which is almost always required, especially for clusters + with low/moderate memory to storage ratio.</description> + </property> + <property> + <name>hbase.regionserver.minorcompaction.pagecache.drop</name> + <value>true</value> + <description>Specifies whether to drop pages read/written into the system page cache by + minor compactions. Setting it to true helps prevent minor compactions from + polluting the page cache, which is most beneficial on clusters with low + memory to storage ratio or very write heavy clusters. You may want to set it to + false under moderate to low write workload when bulk of the reads are + on the most recently written data.</description> + </property> + <property> <name>hbase.storescanner.parallel.seek.enable</name> <value>false</value> <description> http://git-wip-us.apache.org/repos/asf/hbase/blob/4b8195f2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java index d1ab800..af8ffe8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java @@ -77,6 +77,15 @@ public abstract class Compactor<T extends CellSink> { /** specify how many days to keep MVCC values during major compaction **/ protected int keepSeqIdPeriod; + // Configs that drive whether we drop page cache behind compactions + protected static final String MAJOR_COMPACTION_DROP_CACHE = + "hbase.regionserver.majorcompaction.pagecache.drop"; + protected static final String MINOR_COMPACTION_DROP_CACHE = + "hbase.regionserver.minorcompaction.pagecache.drop"; + + private boolean dropCacheMajor; + private boolean dropCacheMinor; + //TODO: depending on Store is not good but, realistically, all compactors currently do. Compactor(final Configuration conf, final Store store) { this.conf = conf; @@ -87,6 +96,8 @@ public abstract class Compactor<T extends CellSink> { Compression.Algorithm.NONE : this.store.getFamily().getCompactionCompression(); this.keepSeqIdPeriod = Math.max(this.conf.getInt(HConstants.KEEP_SEQID_PERIOD, HConstants.MIN_KEEP_SEQID_PERIOD), HConstants.MIN_KEEP_SEQID_PERIOD); + this.dropCacheMajor = conf.getBoolean(MAJOR_COMPACTION_DROP_CACHE, true); + this.dropCacheMinor = conf.getBoolean(MINOR_COMPACTION_DROP_CACHE, true); } public interface CellSink { @@ -267,6 +278,13 @@ public abstract class Compactor<T extends CellSink> { List<StoreFileScanner> scanners; Collection<StoreFile> readersToClose; T writer = null; + boolean dropCache; + if (request.isMajor() || request.isAllFiles()) { + dropCache = this.dropCacheMajor; + } else { + dropCache = this.dropCacheMinor; + } + if (this.conf.getBoolean("hbase.regionserver.compaction.private.readers", true)) { // clone all StoreFiles, so we'll do the compaction on a independent copy of StoreFiles, // HFiles, and their readers @@ -274,12 +292,10 @@ public abstract class Compactor<T extends CellSink> { for (StoreFile f : request.getFiles()) { readersToClose.add(f.cloneForReader()); } - scanners = createFileScanners(readersToClose, smallestReadPoint, - store.throttleCompaction(request.getSize())); + scanners = createFileScanners(readersToClose, smallestReadPoint, dropCache); } else { readersToClose = Collections.emptyList(); - scanners = createFileScanners(request.getFiles(), smallestReadPoint, - store.throttleCompaction(request.getSize())); + scanners = createFileScanners(request.getFiles(), smallestReadPoint, dropCache); } InternalScanner scanner = null; boolean finished = false; @@ -301,7 +317,7 @@ public abstract class Compactor<T extends CellSink> { smallestReadPoint = Math.min(fd.minSeqIdToKeep, smallestReadPoint); cleanSeqId = true; } - writer = sinkFactory.createWriter(scanner, fd, store.throttleCompaction(request.getSize())); + writer = sinkFactory.createWriter(scanner, fd, dropCache); finished = performCompaction(scanner, writer, smallestReadPoint, cleanSeqId, throughputController); if (!finished) {
