This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push: new 5f71c48 HBASE-22796 [HBCK2] Add fix of overlaps to fixMeta hbck Service Signed-off-by: Sakthi <sak...@apache.org> 5f71c48 is described below commit 5f71c487e79f311907e6d39f857ccc7e1ccd0884 Author: stack <st...@apache.org> AuthorDate: Fri Sep 6 14:56:05 2019 +0200 HBASE-22796 [HBCK2] Add fix of overlaps to fixMeta hbck Service Signed-off-by: Sakthi <sak...@apache.org> --- .../org/apache/hadoop/hbase/client/RegionInfo.java | 3 + .../apache/hadoop/hbase/master/CatalogJanitor.java | 46 +++++-- .../org/apache/hadoop/hbase/master/MetaFixer.java | 134 ++++++++++++++++---- .../hbase/master/assignment/RegionStateNode.java | 2 +- .../apache/hadoop/hbase/master/TestMetaFixer.java | 72 +++++++++-- .../hbase/master/TestMetaFixerNoCluster.java | 141 +++++++++++++++++++++ 6 files changed, 348 insertions(+), 50 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java index 8056900..836578c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java @@ -794,6 +794,9 @@ public interface RegionInfo { * @see #isDegenerate() */ default boolean isOverlap(RegionInfo other) { + if (other == null) { + return false; + } if (!getTable().equals(other.getTable())) { return false; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java index 81e5825..ec75e0b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java @@ -457,6 +457,12 @@ public class CatalogJanitor extends ScheduledChore { return this.holes; } + /** + * @return Overlap pairs found as we scanned hbase:meta; ordered by hbase:meta + * table sort. Pairs of overlaps may have overlap with subsequent pairs. + * @see MetaFixer#calculateMerges(int, List) where we aggregate overlaps + * for a single 'merge' call. + */ public List<Pair<RegionInfo, RegionInfo>> getOverlaps() { return this.overlaps; } @@ -479,20 +485,20 @@ public class CatalogJanitor extends ScheduledChore { @Override public String toString() { - StringBuffer sb = new StringBuffer(); + StringBuilder sb = new StringBuilder(); for (Pair<RegionInfo, RegionInfo> p: this.holes) { if (sb.length() > 0) { sb.append(", "); } - sb.append("hole=" + p.getFirst().getRegionNameAsString() + "/" + - p.getSecond().getRegionNameAsString()); + sb.append("hole=").append(p.getFirst().getRegionNameAsString()).append("/"). + append(p.getSecond().getRegionNameAsString()); } for (Pair<RegionInfo, RegionInfo> p: this.overlaps) { if (sb.length() > 0) { sb.append(", "); } - sb.append("overlap=" + p.getFirst().getRegionNameAsString() + "/" + - p.getSecond().getRegionNameAsString()); + sb.append("overlap=").append(p.getFirst().getRegionNameAsString()).append("/"). + append(p.getSecond().getRegionNameAsString()); } for (byte [] r: this.emptyRegionInfo) { if (sb.length() > 0) { @@ -534,6 +540,16 @@ public class CatalogJanitor extends ScheduledChore { */ private RegionInfo previous = null; + /** + * Keep account of the highest end key seen as we move through hbase:meta. + * Usually, the current RegionInfo has the highest end key but if an overlap, + * this may no longer hold. An overlap may be a region with startkey 'd' and + * endkey 'g'. The next region in meta may be 'e' to 'f' and then 'f' to 'g'. + * Looking at previous and current meta row, we won't know about the 'd' to 'g' + * overlap unless we keep a running 'highest-endpoint-seen'. + */ + private RegionInfo highestEndKeyRegionInfo = null; + ReportMakingVisitor(MasterServices services) { this.services = services; } @@ -587,7 +603,7 @@ public class CatalogJanitor extends ScheduledChore { MetaTableAccessor.getRegionInfoColumn()); } else { ri = locations.getDefaultRegionLocation().getRegion(); - checkServer(metaTableRow.getRow(), locations); + checkServer(locations); } if (ri == null) { @@ -615,13 +631,22 @@ public class CatalogJanitor extends ScheduledChore { if (!this.previous.isNext(ri)) { if (this.previous.isOverlap(ri)) { addOverlap(this.previous, ri); + } else if (ri.isOverlap(this.highestEndKeyRegionInfo)) { + // We may have seen a region a few rows back that overlaps this one. + addOverlap(this.highestEndKeyRegionInfo, ri); } else { addHole(this.previous, ri); } + } else if (ri.isOverlap(this.highestEndKeyRegionInfo)) { + // We may have seen a region a few rows back that overlaps this one + // even though it properly 'follows' the region just before. + addOverlap(this.highestEndKeyRegionInfo, ri); } } } this.previous = ri; + this.highestEndKeyRegionInfo = + MetaFixer.getRegionInfoWithLargestEndKey(ri, this.highestEndKeyRegionInfo); return ri; } @@ -658,7 +683,7 @@ public class CatalogJanitor extends ScheduledChore { /** * Run through referenced servers and save off unknown and the dead. */ - private void checkServer(byte [] metaTableRow, RegionLocations locations) { + private void checkServer(RegionLocations locations) { if (this.services == null) { // Can't do this test if no services. return; @@ -672,6 +697,11 @@ public class CatalogJanitor extends ScheduledChore { if (sn == null) { continue; } + if (location.getRegion() == null) { + LOG.warn("Empty RegionInfo in {}", location); + // This should never happen but if it does, will mess up below. + continue; + } // skip the offline regions which belong to disabled table. if (isTableDisabled(location.getRegion())) { continue; @@ -680,7 +710,7 @@ public class CatalogJanitor extends ScheduledChore { isServerKnownAndOnline(sn); switch (state) { case UNKNOWN: - this.report.unknownServers.add(new Pair(location.getRegion(), sn)); + this.report.unknownServers.add(new Pair<>(location.getRegion(), sn)); break; default: diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetaFixer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetaFixer.java index d94e4bd..3477ec6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetaFixer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetaFixer.java @@ -18,8 +18,12 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; -import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; @@ -28,6 +32,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.exceptions.MergeRegionException; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -36,20 +41,32 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; + + /** * Server-side fixing of bad or inconsistent state in hbase:meta. * Distinct from MetaTableAccessor because {@link MetaTableAccessor} is about low-level * manipulations driven by the Master. This class MetaFixer is - * employed by the Master and it 'knows' about holes and orphan + * employed by the Master and it 'knows' about holes and orphans * and encapsulates their fixing on behalf of the Master. */ @InterfaceAudience.Private class MetaFixer { private static final Logger LOG = LoggerFactory.getLogger(MetaFixer.class); + private static final String MAX_MERGE_COUNT_KEY = "hbase.master.metafixer.max.merge.count"; + private static final int MAX_MERGE_COUNT_DEFAULT = 10; private final MasterServices masterServices; + /** + * Maximum for many regions to merge at a time. + */ + private final int maxMergeCount; MetaFixer(MasterServices masterServices) { this.masterServices = masterServices; + this.maxMergeCount = this.masterServices.getConfiguration(). + getInt(MAX_MERGE_COUNT_KEY, MAX_MERGE_COUNT_DEFAULT); } void fix() throws IOException { @@ -66,14 +83,12 @@ class MetaFixer { /** * If hole, it papers it over by adding a region in the filesystem and to hbase:meta. * Does not assign. - * @return True if we fixed any 'holes'. */ - boolean fixHoles(CatalogJanitor.Report report) throws IOException { - boolean result = false; + void fixHoles(CatalogJanitor.Report report) throws IOException { List<Pair<RegionInfo, RegionInfo>> holes = report.getHoles(); if (holes.isEmpty()) { LOG.debug("No holes."); - return result; + return; } for (Pair<RegionInfo, RegionInfo> p: holes) { RegionInfo ri = getHoleCover(p); @@ -86,11 +101,10 @@ class MetaFixer { // in hbase:meta (if the below fails). Should be able to rerun the fix. // The second call to createRegionDir will just go through. Idempotent. Put put = MetaTableAccessor.makePutFromRegionInfo(ri, HConstants.LATEST_TIMESTAMP); - MetaTableAccessor.putsToMetaTable(this.masterServices.getConnection(), Arrays.asList(put)); + MetaTableAccessor.putsToMetaTable(this.masterServices.getConnection(), + Collections.singletonList(put)); LOG.info("Fixed hole by adding {}; region is NOT assigned (assign to online).", ri); - result = true; } - return result; } /** @@ -136,28 +150,94 @@ class MetaFixer { return RegionInfoBuilder.newBuilder(tn).setStartKey(start).setEndKey(end).build(); } - boolean fixOverlaps(CatalogJanitor.Report report) throws IOException { - boolean result = false; - List<Pair<RegionInfo, RegionInfo>> overlaps = report.getOverlaps(); + /** + * Fix overlaps noted in CJ consistency report. + */ + void fixOverlaps(CatalogJanitor.Report report) throws IOException { + for (Set<RegionInfo> regions: calculateMerges(maxMergeCount, report.getOverlaps())) { + RegionInfo [] regionsArray = regions.toArray(new RegionInfo [] {}); + try { + this.masterServices.mergeRegions(regionsArray, + false, HConstants.NO_NONCE, HConstants.NO_NONCE); + } catch (MergeRegionException mre) { + LOG.warn("Failed overlap fix of {}", regionsArray, mre); + } + } + } + + /** + * Run through <code>overlaps</code> and return a list of merges to run. + * Presumes overlaps are ordered (which they are coming out of the CatalogJanitor + * consistency report). + * @param maxMergeCount Maximum regions to merge at a time (avoid merging + * 100k regions in one go!) + */ + @VisibleForTesting + static List<SortedSet<RegionInfo>> calculateMerges(int maxMergeCount, + List<Pair<RegionInfo, RegionInfo>> overlaps) { if (overlaps.isEmpty()) { LOG.debug("No overlaps."); - return result; + return Collections.emptyList(); } - for (Pair<RegionInfo, RegionInfo> p: overlaps) { - RegionInfo ri = getHoleCover(p); - if (ri == null) { - continue; + List<SortedSet<RegionInfo>> merges = new ArrayList<>(); + SortedSet<RegionInfo> currentMergeSet = new TreeSet<>(); + RegionInfo regionInfoWithlargestEndKey = null; + for (Pair<RegionInfo, RegionInfo> pair: overlaps) { + if (regionInfoWithlargestEndKey != null) { + if (!isOverlap(regionInfoWithlargestEndKey, pair) || + currentMergeSet.size() >= maxMergeCount) { + merges.add(currentMergeSet); + currentMergeSet = new TreeSet<>(); + } } - Configuration configuration = this.masterServices.getConfiguration(); - HRegion.createRegionDir(configuration, ri, FSUtils.getRootDir(configuration)); - // If an error here, then we'll have a region in the filesystem but not - // in hbase:meta (if the below fails). Should be able to rerun the fix. - // The second call to createRegionDir will just go through. Idempotent. - Put put = MetaTableAccessor.makePutFromRegionInfo(ri, HConstants.LATEST_TIMESTAMP); - MetaTableAccessor.putsToMetaTable(this.masterServices.getConnection(), Arrays.asList(put)); - LOG.info("Fixed hole by adding {}; region is NOT assigned (assign to online).", ri); - result = true; + currentMergeSet.add(pair.getFirst()); + currentMergeSet.add(pair.getSecond()); + regionInfoWithlargestEndKey = getRegionInfoWithLargestEndKey( + getRegionInfoWithLargestEndKey(pair.getFirst(), pair.getSecond()), + regionInfoWithlargestEndKey); + } + merges.add(currentMergeSet); + return merges; + } + + /** + * @return Either <code>a</code> or <code>b</code>, whichever has the + * endkey that is furthest along in the Table. + */ + @VisibleForTesting + static RegionInfo getRegionInfoWithLargestEndKey(RegionInfo a, RegionInfo b) { + if (a == null) { + // b may be null. + return b; + } + if (b == null) { + // Both are null. The return is not-defined. + return a; + } + if (!a.getTable().equals(b.getTable())) { + // This is an odd one. This should be the right answer. + return b; + } + if (a.isLast()) { + return a; + } + if (b.isLast()) { + return b; + } + int compare = Bytes.compareTo(a.getEndKey(), b.getEndKey()); + return compare == 0 || compare > 0? a: b; + } + + /** + * @return True if an overlap found between passed in <code>ri</code> and + * the <code>pair</code>. Does NOT check the pairs themselves overlap. + */ + @VisibleForTesting + static boolean isOverlap(RegionInfo ri, Pair<RegionInfo, RegionInfo> pair) { + if (ri == null || pair == null) { + // Can't be an overlap in either of these cases. + return false; } - return result; + return ri.isOverlap(pair.getFirst()) || ri.isOverlap(pair.getSecond()); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNode.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNode.java index 2a46dfd..04f42ee 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNode.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateNode.java @@ -290,7 +290,7 @@ public class RegionStateNode implements Comparable<RegionStateNode> { RegionInfo ri = getRegionInfo(); State s = state; if (s != State.OPEN) { - throw new DoNotRetryRegionException(ri.getEncodedName() + " is no OPEN; state=" + s); + throw new DoNotRetryRegionException(ri.getEncodedName() + " is not OPEN; state=" + s); } if (ri.isSplitParent()) { throw new DoNotRetryRegionException( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixer.java index 2bffe31..aba00fe 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixer.java @@ -17,7 +17,10 @@ */ package org.apache.hadoop.hbase.master; +import static org.junit.Assert.assertTrue; + import java.io.IOException; +import java.util.Collections; import java.util.List; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -26,8 +29,11 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.util.Threads; + import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -36,9 +42,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - @Category({MasterTests.class, LargeTests.class}) public class TestMetaFixer { @@ -47,7 +50,6 @@ public class TestMetaFixer { HBaseClassTestRule.forClass(TestMetaFixer.class); @Rule public TestName name = new TestName(); - private static final Logger LOG = LoggerFactory.getLogger(TestMetaFixer.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -75,21 +77,20 @@ public class TestMetaFixer { MasterServices services = TEST_UTIL.getHBaseCluster().getMaster(); services.getCatalogJanitor().scan(); CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport(); - Assert.assertTrue(report.isEmpty()); + assertTrue(report.isEmpty()); int originalCount = ris.size(); // Remove first, last and middle region. See if hole gets plugged. Table has 26 regions. deleteRegion(services, ris.get(ris.size() -1)); deleteRegion(services, ris.get(3)); deleteRegion(services, ris.get(0)); - ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn); services.getCatalogJanitor().scan(); report = services.getCatalogJanitor().getLastReport(); Assert.assertEquals(report.toString(), 3, report.getHoles().size()); MetaFixer fixer = new MetaFixer(services); - Assert.assertTrue(fixer.fixHoles(report)); + fixer.fixHoles(report); services.getCatalogJanitor().scan(); report = services.getCatalogJanitor().getLastReport(); - Assert.assertTrue(report.toString(), report.isEmpty()); + assertTrue(report.toString(), report.isEmpty()); // Disable and reenable so the added regions get reassigned. TEST_UTIL.getAdmin().disableTable(tn); TEST_UTIL.getAdmin().enableTable(tn); @@ -110,18 +111,61 @@ public class TestMetaFixer { List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn); MasterServices services = TEST_UTIL.getHBaseCluster().getMaster(); services.getCatalogJanitor().scan(); - CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport(); - int originalCount = ris.size(); deleteRegion(services, ris.get(0)); services.getCatalogJanitor().scan(); - report = services.getCatalogJanitor().getLastReport(); + CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport(); ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn); - Assert.assertTrue(ris.isEmpty()); + assertTrue(ris.isEmpty()); MetaFixer fixer = new MetaFixer(services); - Assert.assertFalse(fixer.fixHoles(report)); + fixer.fixHoles(report); report = services.getCatalogJanitor().getLastReport(); - Assert.assertTrue(report.isEmpty()); + assertTrue(report.isEmpty()); ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn); Assert.assertEquals(0, ris.size()); } + + private static void makeOverlap(MasterServices services, RegionInfo a, RegionInfo b) + throws IOException { + RegionInfo overlapRegion = RegionInfoBuilder.newBuilder(a.getTable()). + setStartKey(a.getStartKey()). + setEndKey(b.getEndKey()). + build(); + MetaTableAccessor.putsToMetaTable(services.getConnection(), + Collections.singletonList(MetaTableAccessor.makePutFromRegionInfo(overlapRegion, + System.currentTimeMillis()))); + // TODO: Add checks at assign time to PREVENT being able to assign over existing assign. + services.getAssignmentManager().assign(overlapRegion); + } + + @Test + public void testOverlap() throws IOException { + TableName tn = TableName.valueOf(this.name.getMethodName()); + TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY); + List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn); + assertTrue(ris.size() > 5); + MasterServices services = TEST_UTIL.getHBaseCluster().getMaster(); + services.getCatalogJanitor().scan(); + CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport(); + assertTrue(report.isEmpty()); + // Make a simple overlap spanning second and third region. + makeOverlap(services, ris.get(1), ris.get(3)); + makeOverlap(services, ris.get(2), ris.get(3)); + makeOverlap(services, ris.get(2), ris.get(4)); + Threads.sleep(10000); + services.getCatalogJanitor().scan(); + report = services.getCatalogJanitor().getLastReport(); + Assert.assertEquals(6, report.getOverlaps().size()); + Assert.assertEquals(1, MetaFixer.calculateMerges(10, report.getOverlaps()).size()); + MetaFixer fixer = new MetaFixer(services); + fixer.fixOverlaps(report); + while (true) { + services.getCatalogJanitor().scan(); + report = services.getCatalogJanitor().getLastReport(); + if (report.isEmpty()) { + break; + } + Threads.sleep(10); + } + assertTrue(report.toString(), report.isEmpty()); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixerNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixerNoCluster.java new file mode 100644 index 0000000..5b80f89 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixerNoCluster.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.SortedSet; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; + +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + + + +/** + * Test small utility methods inside {@link MetaFixer}. + * For cluster tests see {@link TestMetaFixer} + */ +@Category({MasterTests.class, SmallTests.class}) +public class TestMetaFixerNoCluster { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMetaFixerNoCluster.class); + private static byte [] A = Bytes.toBytes("a"); + private static byte [] B = Bytes.toBytes("b"); + private static byte [] C = Bytes.toBytes("c"); + private static byte [] D = Bytes.toBytes("d"); + private static RegionInfo ALL = RegionInfoBuilder.FIRST_META_REGIONINFO; + private static RegionInfo _ARI = RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME). + setEndKey(A).build(); + private static RegionInfo _BRI = RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME). + setEndKey(B).build(); + private static RegionInfo ABRI = RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME). + setStartKey(A).setEndKey(B).build(); + private static RegionInfo ACRI = + org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME). + setStartKey(A).setEndKey(C).build(); + private static RegionInfo CDRI = + org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME). + setStartKey(C).setEndKey(D).build(); + private static RegionInfo ADRI = + org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME). + setStartKey(A).setEndKey(D).build(); + private static RegionInfo D_RI = + org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME). + setStartKey(D).build(); + private static RegionInfo C_RI = + org.apache.hadoop.hbase.client.RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME). + setStartKey(C).build(); + + @Test + public void testGetRegionInfoWithLargestEndKey() { + assertTrue(MetaFixer.getRegionInfoWithLargestEndKey(_ARI, _BRI).equals(_BRI)); + assertTrue(MetaFixer.getRegionInfoWithLargestEndKey(C_RI, D_RI).equals(C_RI)); + assertTrue(MetaFixer.getRegionInfoWithLargestEndKey(ABRI, CDRI).equals(CDRI)); + assertTrue(MetaFixer.getRegionInfoWithLargestEndKey(null, CDRI).equals(CDRI)); + assertTrue(MetaFixer.getRegionInfoWithLargestEndKey(null, null) == null); + } + + @Test + public void testIsOverlap() { + assertTrue(MetaFixer.isOverlap(_BRI, new Pair<RegionInfo, RegionInfo>(ABRI, ACRI))); + assertFalse(MetaFixer.isOverlap(_ARI, new Pair<RegionInfo, RegionInfo>(C_RI, D_RI))); + assertTrue(MetaFixer.isOverlap(ADRI, new Pair<RegionInfo, RegionInfo>(CDRI, C_RI))); + assertFalse(MetaFixer.isOverlap(_BRI, new Pair<RegionInfo, RegionInfo>(CDRI, C_RI))); + } + + @Test + public void testCalculateMergesNoAggregation() { + List<Pair<RegionInfo, RegionInfo>> overlaps = new ArrayList<>(); + overlaps.add(new Pair<RegionInfo, RegionInfo>(_ARI, _BRI)); + overlaps.add(new Pair<RegionInfo, RegionInfo>(C_RI, D_RI)); + List<SortedSet<RegionInfo>> merges = MetaFixer.calculateMerges(10, overlaps); + assertEquals(2, merges.size()); + assertEquals(2, merges.get(0).size()); + assertEquals(2, merges.get(1).size()); + } + + @Test + public void testCalculateMergesAggregation() { + List<Pair<RegionInfo, RegionInfo>> overlaps = new ArrayList<>(); + overlaps.add(new Pair<RegionInfo, RegionInfo>(ALL, D_RI)); + overlaps.add(new Pair<RegionInfo, RegionInfo>(_ARI, _BRI)); + overlaps.add(new Pair<RegionInfo, RegionInfo>(C_RI, D_RI)); + List<SortedSet<RegionInfo>> merges = MetaFixer.calculateMerges(10, overlaps); + assertEquals(1, merges.size()); + assertEquals(5, merges.get(0).size()); + } + + @Test + public void testCalculateMergesNoRepeatOfRegionNames() { + List<Pair<RegionInfo, RegionInfo>> overlaps = new ArrayList<>(); + overlaps.add(new Pair<RegionInfo, RegionInfo>(_BRI, ABRI)); + overlaps.add(new Pair<RegionInfo, RegionInfo>(ABRI, ADRI)); + List<SortedSet<RegionInfo>> merges = MetaFixer.calculateMerges(10, overlaps); + assertEquals(1, merges.size()); + // There should be three regions to merge, not four. + assertEquals(3, merges.get(0).size()); + } + + @Test + public void testCalculateMergesRespectsMax() { + List<Pair<RegionInfo, RegionInfo>> overlaps = new ArrayList<>(); + overlaps.add(new Pair<RegionInfo, RegionInfo>(_BRI, ABRI)); + overlaps.add(new Pair<RegionInfo, RegionInfo>(ABRI, ADRI)); + overlaps.add(new Pair<RegionInfo, RegionInfo>(C_RI, D_RI)); + List<SortedSet<RegionInfo>> merges = MetaFixer.calculateMerges(3, overlaps); + assertEquals(2, merges.size()); + // There should be three regions to merge, not four. + assertEquals(3, merges.get(0).size()); + assertEquals(2, merges.get(1).size()); + } +}