Repository: hbase Updated Branches: refs/heads/master c73180d45 -> 0d6b872d9
HBASE-17241 Avoid compacting already compacted mob files with _del files (huaxiang sun) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d6b872d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d6b872d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d6b872d Branch: refs/heads/master Commit: 0d6b872d9dca4c040a172eed7b0153ece36c0a0d Parents: c73180d Author: tedyu <[email protected]> Authored: Sun Dec 4 20:23:06 2016 -0800 Committer: tedyu <[email protected]> Committed: Sun Dec 4 20:23:06 2016 -0800 ---------------------------------------------------------------------- .../PartitionedMobCompactionRequest.java | 4 +++ .../compactions/PartitionedMobCompactor.java | 20 +++++++++++ .../TestPartitionedMobCompactor.java | 38 +++++++++++++++++--- 3 files changed, 57 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6b872d/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java index e288571..665b5e2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java @@ -86,6 +86,10 @@ public class PartitionedMobCompactionRequest extends MobCompactionRequest { public List<FileStatus> listFiles() { return Collections.unmodifiableList(files); } + + public int getFileCount () { + return files.size(); + } } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6b872d/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java index 33f5e71..2f7d961 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -187,6 +188,25 @@ public class PartitionedMobCompactor extends MobCompactor { selectedFileCount++; } } + + /* + * If it is not a major mob compaction with del files, and the file number in Partition is 1, + * remove the partition from filesToCompact list to avoid re-compacting files which has been + * compacted with del files. + */ + if (!allFiles && (allDelFiles.size() > 0)) { + Iterator<Map.Entry<CompactionPartitionId, CompactionPartition>> it = + filesToCompact.entrySet().iterator(); + + while(it.hasNext()) { + Map.Entry<CompactionPartitionId, CompactionPartition> entry = it.next(); + if (entry.getValue().getFileCount() == 1) { + it.remove(); + --selectedFileCount; + } + } + } + PartitionedMobCompactionRequest request = new PartitionedMobCompactionRequest( filesToCompact.values(), allDelFiles); if (candidates.size() == (allDelFiles.size() + selectedFileCount + irrelevantFileCount)) { http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6b872d/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java index 7da8544..c112289 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java @@ -106,11 +106,23 @@ public class TestPartitionedMobCompactor { @Test public void testCompactionSelectWithAllFiles() throws Exception { String tableName = "testCompactionSelectWithAllFiles"; + // If there is only 1 file, it will not be compacted with _del files, so + // It wont be CompactionType.ALL_FILES in this case, do not create with _del files. testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD, - CompactionType.ALL_FILES, false); + CompactionType.ALL_FILES, false, false); } @Test + public void testCompactionSelectToAvoidCompactOneFileWithDelete() throws Exception { + String tableName = "testCompactionSelectToAvoidCompactOneFileWithDelete"; + // If there is only 1 file, it will not be compacted with _del files, so + // It wont be CompactionType.ALL_FILES in this case, and expected compact file count will be 0. + testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD, + CompactionType.PART_FILES, false); + } + + + @Test public void testCompactionSelectWithPartFiles() throws Exception { String tableName = "testCompactionSelectWithPartFiles"; testCompactionAtMergeSize(tableName, 4000, CompactionType.PART_FILES, false); @@ -125,20 +137,36 @@ public class TestPartitionedMobCompactor { private void testCompactionAtMergeSize(final String tableName, final long mergeSize, final CompactionType type, final boolean isForceAllFiles) throws Exception { + testCompactionAtMergeSize(tableName, mergeSize, type, isForceAllFiles, true); + } + + private void testCompactionAtMergeSize(final String tableName, + final long mergeSize, final CompactionType type, final boolean isForceAllFiles, + final boolean createDelFiles) + throws Exception { resetConf(); init(tableName); int count = 10; // create 10 mob files. createStoreFiles(basePath, family, qf, count, Type.Put); - // create 10 del files - createStoreFiles(basePath, family, qf, count, Type.Delete); + + if (createDelFiles) { + // create 10 del files + createStoreFiles(basePath, family, qf, count, Type.Delete); + } + listFiles(); List<String> expectedStartKeys = new ArrayList<>(); for(FileStatus file : mobFiles) { if(file.getLen() < mergeSize) { String fileName = file.getPath().getName(); String startKey = fileName.substring(0, 32); - expectedStartKeys.add(startKey); + + // If it is not an major mob compaction and del files are there, + // these mob files wont be compacted. + if (isForceAllFiles || !createDelFiles) { + expectedStartKeys.add(startKey); + } } } // set the mob compaction mergeable threshold @@ -150,7 +178,7 @@ public class TestPartitionedMobCompactor { public void testCompactDelFilesWithDefaultBatchSize() throws Exception { String tableName = "testCompactDelFilesWithDefaultBatchSize"; testCompactDelFilesAtBatchSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE, - MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT); + MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT); } @Test
