HBASE-19751 Use RegionInfo directly instead of an identifier and a namespace when getting WAL
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/814d08a2 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/814d08a2 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/814d08a2 Branch: refs/heads/HBASE-19397-branch-2 Commit: 814d08a2d663ccdbacb33e84475713a72bf3726a Parents: 0885fe1 Author: zhangduo <zhang...@apache.org> Authored: Thu Jan 11 15:47:08 2018 +0800 Committer: zhangduo <zhang...@apache.org> Committed: Thu Jan 11 15:48:33 2018 +0800 ---------------------------------------------------------------------- .../hbase/mapreduce/TestWALRecordReader.java | 50 +++---- .../hbase/regionserver/HRegionServer.java | 14 +- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 6 +- .../hadoop/hbase/wal/AbstractFSWALProvider.java | 3 +- .../hadoop/hbase/wal/DisabledWALProvider.java | 2 +- .../hbase/wal/RegionGroupingProvider.java | 40 +++--- .../org/apache/hadoop/hbase/wal/WALFactory.java | 64 ++++----- .../apache/hadoop/hbase/wal/WALProvider.java | 17 ++- .../hadoop/hbase/HBaseTestingUtility.java | 5 +- .../hbase/coprocessor/TestWALObserver.java | 117 +++++++---------- .../regionserver/TestCacheOnWriteInSchema.java | 52 ++++---- .../TestCompactionArchiveConcurrentClose.java | 31 ++--- .../TestCompactionArchiveIOException.java | 42 +++--- .../hbase/regionserver/TestDefaultMemStore.java | 45 +++---- .../hbase/regionserver/TestHMobStore.java | 3 +- .../hadoop/hbase/regionserver/TestHRegion.java | 13 +- .../regionserver/TestHRegionReplayEvents.java | 43 +++--- .../hadoop/hbase/regionserver/TestHStore.java | 14 +- .../TestStoreFileRefresherChore.java | 51 ++++---- .../TestWALMonotonicallyIncreasingSeqId.java | 2 +- .../wal/AbstractTestLogRolling.java | 9 +- .../hbase/regionserver/wal/TestDurability.java | 93 +++++++------ .../regionserver/wal/TestLogRollAbort.java | 39 +++--- .../wal/TestLogRollingNoCluster.java | 17 +-- .../wal/TestWALActionsListener.java | 25 ++-- .../TestReplicationSourceManager.java | 2 +- .../regionserver/TestWALEntryStream.java | 13 +- .../apache/hadoop/hbase/wal/IOTestProvider.java | 48 ++++--- .../wal/TestBoundedRegionGroupingStrategy.java | 16 ++- .../hadoop/hbase/wal/TestFSHLogProvider.java | 130 ++++++++----------- .../apache/hadoop/hbase/wal/TestSecureWAL.java | 22 +--- .../apache/hadoop/hbase/wal/TestWALFactory.java | 127 +++++++----------- .../apache/hadoop/hbase/wal/TestWALMethods.java | 2 +- .../hbase/wal/TestWALReaderOnSecureWAL.java | 25 ++-- .../apache/hadoop/hbase/wal/TestWALRootDir.java | 35 +++-- .../apache/hadoop/hbase/wal/TestWALSplit.java | 37 +++--- .../hbase/wal/WALPerformanceEvaluation.java | 47 ++++--- 37 files changed, 591 insertions(+), 710 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java ---------------------------------------------------------------------- diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java index 18bb135..c8db903 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java @@ -24,30 +24,28 @@ import static org.junit.Assert.assertTrue; import java.util.List; import java.util.NavigableMap; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.mapreduce.WALInputFormat.WALKeyRecordReader; import org.apache.hadoop.hbase.mapreduce.WALInputFormat.WALRecordReader; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; -import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.WAL; +import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; +import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALKeyImpl; -import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.MapReduceTestUtil; import org.junit.AfterClass; @@ -61,7 +59,7 @@ import org.slf4j.LoggerFactory; /** * JUnit tests for the WALRecordReader */ -@Category({MapReduceTests.class, MediumTests.class}) +@Category({ MapReduceTests.class, MediumTests.class }) public class TestWALRecordReader { private static final Logger LOG = LoggerFactory.getLogger(TestWALRecordReader.class); private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -74,11 +72,9 @@ public class TestWALRecordReader { static final TableName tableName = TableName.valueOf(getName()); private static final byte [] rowName = tableName.getName(); // visible for TestHLogRecordReader - static final HRegionInfo info = new HRegionInfo(tableName, - Bytes.toBytes(""), Bytes.toBytes(""), false); - private static final byte [] family = Bytes.toBytes("column"); - private static final byte [] value = Bytes.toBytes("value"); - private static HTableDescriptor htd; + static final RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); + private static final byte[] family = Bytes.toBytes("column"); + private static final byte[] value = Bytes.toBytes("value"); private static Path logDir; protected MultiVersionConcurrencyControl mvcc; protected static NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); @@ -93,6 +89,7 @@ public class TestWALRecordReader { walFs.delete(walRootDir, true); mvcc = new MultiVersionConcurrencyControl(); } + @BeforeClass public static void setUpBeforeClass() throws Exception { // Make block sizes small. @@ -108,9 +105,6 @@ public class TestWALRecordReader { walRootDir = TEST_UTIL.createWALRootDir(); walFs = FSUtils.getWALFileSystem(conf); logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME); - - htd = new HTableDescriptor(tableName); - htd.addFamily(new HColumnDescriptor(family)); } @AfterClass @@ -127,7 +121,7 @@ public class TestWALRecordReader { @Test public void testPartialRead() throws Exception { final WALFactory walfactory = new WALFactory(conf, null, getName()); - WAL log = walfactory.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()); + WAL log = walfactory.getWAL(info); // This test depends on timestamp being millisecond based and the filename of the WAL also // being millisecond based. long ts = System.currentTimeMillis(); @@ -186,9 +180,8 @@ public class TestWALRecordReader { @Test public void testWALRecordReader() throws Exception { final WALFactory walfactory = new WALFactory(conf, null, getName()); - WAL log = walfactory.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()); + WAL log = walfactory.getWAL(info); byte [] value = Bytes.toBytes("value"); - final AtomicLong sequenceId = new AtomicLong(0); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"), System.currentTimeMillis(), value)); @@ -245,7 +238,7 @@ public class TestWALRecordReader { return new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, time, mvcc, scopes); } - protected WALRecordReader getReader() { + private WALRecordReader<WALKey> getReader() { return new WALKeyRecordReader(); } @@ -253,7 +246,7 @@ public class TestWALRecordReader { * Create a new reader from the split, and match the edits against the passed columns. */ private void testSplit(InputSplit split, byte[]... columns) throws Exception { - final WALRecordReader reader = getReader(); + WALRecordReader<WALKey> reader = getReader(); reader.initialize(split, MapReduceTestUtil.createDummyMapTaskAttemptContext(conf)); for (byte[] column : columns) { @@ -262,15 +255,12 @@ public class TestWALRecordReader { if (!Bytes.equals(column, 0, column.length, cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())) { assertTrue( - "expected [" - + Bytes.toString(column) - + "], actual [" - + Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), - cell.getQualifierLength()) + "]", false); + "expected [" + Bytes.toString(column) + "], actual [" + Bytes.toString( + cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()) + "]", + false); } } assertFalse(reader.nextKeyValue()); reader.close(); } - } http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 63451c6..53390bd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -2105,8 +2105,6 @@ public class HRegionServer extends HasThread implements return healthy; } - private static final byte[] UNSPECIFIED_REGION = new byte[]{}; - @Override public List<WAL> getWALs() throws IOException { return walFactory.getWALs(); @@ -2114,17 +2112,7 @@ public class HRegionServer extends HasThread implements @Override public WAL getWAL(RegionInfo regionInfo) throws IOException { - WAL wal; - // _ROOT_ and hbase:meta regions have separate WAL. - if (regionInfo != null && regionInfo.isMetaRegion() - && regionInfo.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { - wal = walFactory.getMetaWAL(regionInfo.getEncodedNameAsBytes()); - } else if (regionInfo == null) { - wal = walFactory.getWAL(UNSPECIFIED_REGION, null); - } else { - byte[] namespace = regionInfo.getTable().getNamespace(); - wal = walFactory.getWAL(regionInfo.getEncodedNameAsBytes(), namespace); - } + WAL wal = walFactory.getWAL(regionInfo); if (this.walRoller != null) { this.walRoller.addWAL(wal); } http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 5ca1ed6..d0276c0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -1480,9 +1480,9 @@ public class HBaseFsck extends Configured implements Closeable { // unless I pass along via the conf. Configuration confForWAL = new Configuration(c); confForWAL.set(HConstants.HBASE_DIR, rootdir.toString()); - WAL wal = (new WALFactory(confForWAL, - Collections.<WALActionsListener> singletonList(new MetricsWAL()), walFactoryID)) - .getWAL(metaHRI.getEncodedNameAsBytes(), metaHRI.getTable().getNamespace()); + WAL wal = + new WALFactory(confForWAL, Collections.<WALActionsListener> singletonList(new MetricsWAL()), + walFactoryID).getWAL(metaHRI); HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal); MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true); return meta; http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java index 103b05f..74d502e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; import org.slf4j.Logger; @@ -128,7 +129,7 @@ public abstract class AbstractFSWALProvider<T extends AbstractFSWAL<?>> implemen } @Override - public T getWAL(byte[] identifier, byte[] namespace) throws IOException { + public T getWAL(RegionInfo region) throws IOException { T walCopy = wal; if (walCopy == null) { // only lock when need to create wal, and need to lock since http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java index 280d95f..725f9ff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java @@ -74,7 +74,7 @@ class DisabledWALProvider implements WALProvider { } @Override - public WAL getWAL(final byte[] identifier, byte[] namespace) throws IOException { + public WAL getWAL(RegionInfo region) throws IOException { return disabled; } http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java index b8c9484..a3e54a5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java @@ -27,15 +27,17 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; - +import java.util.concurrent.locks.Lock; import org.apache.hadoop.conf.Configuration; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.RegionInfo; // imports for classes still in regionserver.wal import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.IdLock; +import org.apache.hadoop.hbase.util.KeyLocker; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A WAL Provider that returns a WAL per group of regions. @@ -131,7 +133,7 @@ public class RegionGroupingProvider implements WALProvider { /** A group-provider mapping, make sure one-one rather than many-one mapping */ private final ConcurrentMap<String, WALProvider> cached = new ConcurrentHashMap<>(); - private final IdLock createLock = new IdLock(); + private final KeyLocker<String> createLock = new KeyLocker<>(); private RegionGroupingStrategy strategy = null; private WALFactory factory = null; @@ -177,33 +179,39 @@ public class RegionGroupingProvider implements WALProvider { return wals; } - private WAL getWAL(final String group) throws IOException { + private WAL getWAL(String group) throws IOException { WALProvider provider = cached.get(group); if (provider == null) { - IdLock.Entry lockEntry = null; + Lock lock = createLock.acquireLock(group); try { - lockEntry = createLock.getLockEntry(group.hashCode()); provider = cached.get(group); if (provider == null) { provider = createProvider(group); cached.put(group, provider); } } finally { - if (lockEntry != null) { - createLock.releaseLockEntry(lockEntry); - } + lock.unlock(); } } - return provider.getWAL(null, null); + return provider.getWAL(null); } @Override - public WAL getWAL(final byte[] identifier, byte[] namespace) throws IOException { - final String group; + public WAL getWAL(RegionInfo region) throws IOException { + String group; if (META_WAL_PROVIDER_ID.equals(this.providerId)) { group = META_WAL_GROUP_NAME; } else { - group = strategy.group(identifier, namespace); + byte[] id; + byte[] namespace; + if (region != null) { + id = region.getEncodedNameAsBytes(); + namespace = region.getTable().getNamespace(); + } else { + id = HConstants.EMPTY_BYTE_ARRAY; + namespace = null; + } + group = strategy.group(id, namespace); } return getWAL(group); } http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java index 1ed4168..d70b8cd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java @@ -1,5 +1,4 @@ /** - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -16,25 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - package org.apache.hadoop.hbase.wal; -import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; - import java.io.IOException; import java.io.InterruptedIOException; import java.util.Collections; import java.util.List; import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicReference; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.client.RegionInfo; // imports for things that haven't moved from regionserver.wal yet. import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader; @@ -45,6 +37,11 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.LeaseNotRecoveredException; import org.apache.hadoop.hbase.wal.WAL.Reader; import org.apache.hadoop.hbase.wal.WALProvider.Writer; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; /** * Entry point for users of the Write Ahead Log. @@ -91,11 +88,11 @@ public class WALFactory implements WALFileLengthProvider { static final String DEFAULT_META_WAL_PROVIDER = Providers.defaultProvider.name(); final String factoryId; - final WALProvider provider; + private final WALProvider provider; // The meta updates are written to a different wal. If this // regionserver holds meta regions, then this ref will be non-null. // lazily intialized; most RegionServers don't deal with META - final AtomicReference<WALProvider> metaProvider = new AtomicReference<>(); + private final AtomicReference<WALProvider> metaProvider = new AtomicReference<>(); /** * Configuration-specified WAL Reader used when a custom reader is requested @@ -236,32 +233,35 @@ public class WALFactory implements WALFileLengthProvider { return provider.getWALs(); } - /** - * @param identifier may not be null, contents will not be altered - * @param namespace could be null, and will use default namespace if null - */ - public WAL getWAL(final byte[] identifier, final byte[] namespace) throws IOException { - return provider.getWAL(identifier, namespace); + private WALProvider getMetaProvider() throws IOException { + for (;;) { + WALProvider provider = this.metaProvider.get(); + if (provider != null) { + return provider; + } + provider = getProvider(META_WAL_PROVIDER, DEFAULT_META_WAL_PROVIDER, + Collections.<WALActionsListener> singletonList(new MetricsWAL()), + AbstractFSWALProvider.META_WAL_PROVIDER_ID); + if (metaProvider.compareAndSet(null, provider)) { + return provider; + } else { + // someone is ahead of us, close and try again. + provider.close(); + } + } } /** - * @param identifier may not be null, contents will not be altered + * @param region the region which we want to get a WAL for it. Could be null. */ - public WAL getMetaWAL(final byte[] identifier) throws IOException { - WALProvider metaProvider = this.metaProvider.get(); - if (null == metaProvider) { - final WALProvider temp = getProvider(META_WAL_PROVIDER, DEFAULT_META_WAL_PROVIDER, - Collections.<WALActionsListener>singletonList(new MetricsWAL()), - AbstractFSWALProvider.META_WAL_PROVIDER_ID); - if (this.metaProvider.compareAndSet(null, temp)) { - metaProvider = temp; - } else { - // reference must now be to a provider created in another thread. - temp.close(); - metaProvider = this.metaProvider.get(); - } + public WAL getWAL(RegionInfo region) throws IOException { + // use different WAL for hbase:meta + if (region != null && region.isMetaRegion() && + region.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { + return getMetaProvider().getWAL(region); + } else { + return provider.getWAL(region); } - return metaProvider.getWAL(identifier, null); } public Reader createReader(final FileSystem fs, final Path path) throws IOException { http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALProvider.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALProvider.java index c38f419..0586d1d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALProvider.java @@ -22,11 +22,10 @@ import java.io.Closeable; import java.io.IOException; import java.util.List; import java.util.concurrent.CompletableFuture; - import org.apache.hadoop.conf.Configuration; -import org.apache.yetus.audience.InterfaceAudience; -// imports for things that haven't moved from regionserver.wal yet. +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; +import org.apache.yetus.audience.InterfaceAudience; /** * The Write Ahead Log (WAL) stores all durable edits to the HRegion. @@ -48,17 +47,17 @@ public interface WALProvider { * @param listeners may be null * @param providerId differentiate between providers from one factory. may be null */ - void init(final WALFactory factory, final Configuration conf, - final List<WALActionsListener> listeners, final String providerId) throws IOException; + void init(WALFactory factory, Configuration conf, List<WALActionsListener> listeners, + String providerId) throws IOException; /** - * @param identifier may not be null. contents will not be altered. - * @param namespace could be null, and will use default namespace if null + * @param region the region which we want to get a WAL for it. Could be null. * @return a WAL for writing entries for the given region. */ - WAL getWAL(final byte[] identifier, byte[] namespace) throws IOException; + WAL getWAL(RegionInfo region) throws IOException; - /** @return the List of WALs that are used by this server + /** + * @return the List of WALs that are used by this server */ List<WAL> getWALs(); http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 304b3cb..fe8902a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -2310,9 +2310,8 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { Configuration confForWAL = new Configuration(conf); confForWAL.set(HConstants.HBASE_DIR, rootDir.toString()); return (new WALFactory(confForWAL, - Collections.<WALActionsListener>singletonList(new MetricsWAL()), - "hregion-" + RandomStringUtils.randomNumeric(8))). - getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()); + Collections.<WALActionsListener> singletonList(new MetricsWAL()), + "hregion-" + RandomStringUtils.randomNumeric(8))).getWAL(hri); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java index 86a0d39..166dfdd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java @@ -1,5 +1,4 @@ -/* - * +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -16,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.coprocessor; import static org.junit.Assert.assertEquals; @@ -31,7 +29,6 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.TreeMap; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -40,18 +37,19 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost; -import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -61,6 +59,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; +import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitter; @@ -172,17 +171,17 @@ public class TestWALObserver { */ @Test public void testWALObserverWriteToWAL() throws Exception { - final WAL log = wals.getWAL(UNSPECIFIED_REGION, null); + final WAL log = wals.getWAL(null); verifyWritesSeen(log, getCoprocessor(log, SampleRegionWALCoprocessor.class), false); } private void verifyWritesSeen(final WAL log, final SampleRegionWALCoprocessor cp, final boolean seesLegacy) throws Exception { - HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE)); - final HTableDescriptor htd = createBasic3FamilyHTD(Bytes + RegionInfo hri = createBasicHRegionInfo(Bytes.toString(TEST_TABLE)); + TableDescriptor htd = createBasic3FamilyHTD(Bytes .toString(TEST_TABLE)); NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); - for(byte[] fam : htd.getFamiliesKeys()) { + for (byte[] fam : htd.getColumnFamilyNames()) { scopes.put(fam, 0); } Path basedir = new Path(this.hbaseRootDir, Bytes.toString(TEST_TABLE)); @@ -268,14 +267,14 @@ public class TestWALObserver { */ @Test public void testEmptyWALEditAreNotSeen() throws Exception { - final HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE)); - final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE)); - final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); + RegionInfo hri = createBasicHRegionInfo(Bytes.toString(TEST_TABLE)); + TableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE)); + MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); - for(byte[] fam : htd.getFamiliesKeys()) { + for(byte[] fam : htd.getColumnFamilyNames()) { scopes.put(fam, 0); } - WAL log = wals.getWAL(UNSPECIFIED_REGION, null); + WAL log = wals.getWAL(null); try { SampleRegionWALCoprocessor cp = getCoprocessor(log, SampleRegionWALCoprocessor.class); @@ -304,14 +303,14 @@ public class TestWALObserver { public void testWALCoprocessorReplay() throws Exception { // WAL replay is handled at HRegion::replayRecoveredEdits(), which is // ultimately called by HRegion::initialize() - final TableName tableName = TableName.valueOf(currentTest.getMethodName()); - final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(tableName); + TableName tableName = TableName.valueOf(currentTest.getMethodName()); + TableDescriptor htd = getBasic3FamilyHTableDescriptor(tableName); MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); // final HRegionInfo hri = // createBasic3FamilyHRegionInfo(Bytes.toString(tableName)); // final HRegionInfo hri1 = // createBasic3FamilyHRegionInfo(Bytes.toString(tableName)); - final HRegionInfo hri = new HRegionInfo(tableName, null, null); + RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build(); final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName); @@ -321,21 +320,21 @@ public class TestWALObserver { final Configuration newConf = HBaseConfiguration.create(this.conf); // WAL wal = new WAL(this.fs, this.dir, this.oldLogDir, this.conf); - WAL wal = wals.getWAL(UNSPECIFIED_REGION, null); + WAL wal = wals.getWAL(null); // Put p = creatPutWith2Families(TEST_ROW); WALEdit edit = new WALEdit(); long now = EnvironmentEdgeManager.currentTime(); final int countPerFamily = 1000; NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); - for (HColumnDescriptor hcd : htd.getFamilies()) { - scopes.put(hcd.getName(), 0); + for (byte[] fam : htd.getColumnFamilyNames()) { + scopes.put(fam, 0); } - for (HColumnDescriptor hcd : htd.getFamilies()) { - addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, - EnvironmentEdgeManager.getDelegate(), wal, scopes, mvcc); + for (byte[] fam : htd.getColumnFamilyNames()) { + addWALEdits(tableName, hri, TEST_ROW, fam, countPerFamily, + EnvironmentEdgeManager.getDelegate(), wal, scopes, mvcc); } wal.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit, - true); + true); // sync to fs. wal.sync(); @@ -345,14 +344,12 @@ public class TestWALObserver { public Void run() throws Exception { Path p = runWALSplit(newConf); LOG.info("WALSplit path == " + p); - FileSystem newFS = FileSystem.get(newConf); // Make a new wal for new region open. final WALFactory wals2 = new WALFactory(conf, null, ServerName.valueOf(currentTest.getMethodName()+"2", 16010, System.currentTimeMillis()).toString()); - WAL wal2 = wals2.getWAL(UNSPECIFIED_REGION, null);; + WAL wal2 = wals2.getWAL(null); HRegion region = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir, hri, htd, wal2, TEST_UTIL.getHBaseCluster().getRegionServer(0), null); - long seqid2 = region.getOpenSeqNum(); SampleRegionWALCoprocessor cp2 = region.getCoprocessorHost().findCoprocessor(SampleRegionWALCoprocessor.class); @@ -374,13 +371,13 @@ public class TestWALObserver { */ @Test public void testWALObserverLoaded() throws Exception { - WAL log = wals.getWAL(UNSPECIFIED_REGION, null); + WAL log = wals.getWAL(null); assertNotNull(getCoprocessor(log, SampleRegionWALCoprocessor.class)); } @Test public void testWALObserverRoll() throws Exception { - final WAL wal = wals.getWAL(UNSPECIFIED_REGION, null); + final WAL wal = wals.getWAL(null); final SampleRegionWALCoprocessor cp = getCoprocessor(wal, SampleRegionWALCoprocessor.class); cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null); @@ -399,20 +396,12 @@ public class TestWALObserver { return (SampleRegionWALCoprocessor) c; } - /* - * Creates an HRI around an HTD that has <code>tableName</code> and three - * column families named. - * - * @param tableName Name of table to use when we create HTableDescriptor. + /** + * Creates an HRI around an HTD that has <code>tableName</code>. + * @param tableName Name of table to use. */ - private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); - - for (int i = 0; i < TEST_FAMILY.length; i++) { - HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]); - htd.addFamily(a); - } - return new HRegionInfo(htd.getTableName(), null, null, false); + private RegionInfo createBasicHRegionInfo(String tableName) { + return RegionInfoBuilder.newBuilder(TableName.valueOf(tableName)).build(); } /* @@ -463,12 +452,10 @@ public class TestWALObserver { return splits.get(0); } - private static final byte[] UNSPECIFIED_REGION = new byte[]{}; - - private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName, + private void addWALEdits(final TableName tableName, final RegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final WAL wal, final NavigableMap<byte[], Integer> scopes, final MultiVersionConcurrencyControl mvcc) - throws IOException { + throws IOException { String familyStr = Bytes.toString(family); long txid = -1; for (int j = 0; j < count; j++) { @@ -478,33 +465,25 @@ public class TestWALObserver { edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); // uses WALKeyImpl instead of HLogKey on purpose. will only work for tests where we don't care // about legacy coprocessors - txid = wal.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, - ee.currentTime(), mvcc), edit, true); + txid = wal.append(hri, + new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, ee.currentTime(), mvcc), edit, true); } if (-1 != txid) { wal.sync(txid); } } - private HTableDescriptor getBasic3FamilyHTableDescriptor( - final TableName tableName) { - HTableDescriptor htd = new HTableDescriptor(tableName); - - for (int i = 0; i < TEST_FAMILY.length; i++) { - HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]); - htd.addFamily(a); - } - return htd; + private TableDescriptor getBasic3FamilyHTableDescriptor(TableName tableName) { + TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); + Arrays.stream(TEST_FAMILY).map(ColumnFamilyDescriptorBuilder::of) + .forEachOrdered(builder::addColumnFamily); + return builder.build(); } - private HTableDescriptor createBasic3FamilyHTD(final String tableName) { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); - HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a")); - htd.addFamily(a); - HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b")); - htd.addFamily(b); - HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c")); - htd.addFamily(c); - return htd; + private TableDescriptor createBasic3FamilyHTD(String tableName) { + return TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)) + .addColumnFamily(ColumnFamilyDescriptorBuilder.of("a")) + .addColumnFamily(ColumnFamilyDescriptorBuilder.of("b")) + .addColumnFamily(ColumnFamilyDescriptorBuilder.of("c")).build(); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java index 02a21b6..0f4ece6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java @@ -26,16 +26,18 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Random; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.BlockCache; @@ -106,18 +108,19 @@ public class TestCacheOnWriteInSchema { return blockType == blockType1 || blockType == blockType2; } - public void modifyFamilySchema(HColumnDescriptor family) { + public ColumnFamilyDescriptorBuilder modifyFamilySchema(ColumnFamilyDescriptorBuilder builder) { switch (this) { - case DATA_BLOCKS: - family.setCacheDataOnWrite(true); - break; - case BLOOM_BLOCKS: - family.setCacheBloomsOnWrite(true); - break; - case INDEX_BLOCKS: - family.setCacheIndexesOnWrite(true); - break; + case DATA_BLOCKS: + builder.setCacheDataOnWrite(true); + break; + case BLOOM_BLOCKS: + builder.setCacheBloomsOnWrite(true); + break; + case INDEX_BLOCKS: + builder.setCacheIndexesOnWrite(true); + break; } + return builder; } } @@ -158,23 +161,22 @@ public class TestCacheOnWriteInSchema { fs = HFileSystem.get(conf); // Create the schema - HColumnDescriptor hcd = new HColumnDescriptor(family); - hcd.setBloomFilterType(BloomType.ROWCOL); - cowType.modifyFamilySchema(hcd); - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table)); - htd.addFamily(hcd); + ColumnFamilyDescriptor hcd = cowType + .modifyFamilySchema( + ColumnFamilyDescriptorBuilder.newBuilder(family).setBloomFilterType(BloomType.ROWCOL)) + .build(); + TableDescriptor htd = + TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).addColumnFamily(hcd).build(); // Create a store based on the schema - final String id = TestCacheOnWriteInSchema.class.getName(); - final Path logdir = new Path(FSUtils.getRootDir(conf), - AbstractFSWALProvider.getWALDirectoryName(id)); + String id = TestCacheOnWriteInSchema.class.getName(); + Path logdir = new Path(FSUtils.getRootDir(conf), AbstractFSWALProvider.getWALDirectoryName(id)); fs.delete(logdir, true); - HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); + RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); walFactory = new WALFactory(conf, null, id); - region = TEST_UTIL.createLocalHRegion(info, htd, - walFactory.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace())); + region = TEST_UTIL.createLocalHRegion(info, htd, walFactory.getWAL(info)); store = new HStore(region, hcd, conf); } http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java index 95dec3d..d2e1866 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.regionserver; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.IOException; import java.io.InterruptedIOException; @@ -29,17 +28,18 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -85,10 +85,10 @@ public class TestCompactionArchiveConcurrentClose { byte[] col = Bytes.toBytes("c"); byte[] val = Bytes.toBytes("val"); - final TableName tableName = TableName.valueOf(name.getMethodName()); - HTableDescriptor htd = new HTableDescriptor(tableName); - htd.addFamily(new HColumnDescriptor(fam)); - HRegionInfo info = new HRegionInfo(tableName, null, null, false); + TableName tableName = TableName.valueOf(name.getMethodName()); + TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) + .addColumnFamily(ColumnFamilyDescriptorBuilder.of(fam)).build(); + RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); HRegion region = initHRegion(htd, info); RegionServerServices rss = mock(RegionServerServices.class); List<HRegion> regions = new ArrayList<>(); @@ -157,20 +157,17 @@ public class TestCompactionArchiveConcurrentClose { } } - private HRegion initHRegion(HTableDescriptor htd, HRegionInfo info) - throws IOException { + private HRegion initHRegion(TableDescriptor htd, RegionInfo info) throws IOException { Configuration conf = testUtil.getConfiguration(); Path tableDir = FSUtils.getTableDir(testDir, htd.getTableName()); - HRegionFileSystem fs = new WaitingHRegionFileSystem(conf, tableDir.getFileSystem(conf), - tableDir, info); + HRegionFileSystem fs = + new WaitingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir, info); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); final Configuration walConf = new Configuration(conf); FSUtils.setRootDir(walConf, tableDir); final WALFactory wals = new WALFactory(walConf, null, "log_" + info.getEncodedName()); - HRegion region = - new HRegion(fs, wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), - conf, htd, null); + HRegion region = new HRegion(fs, wals.getWAL(info), conf, htd, null); region.initialize(); @@ -180,7 +177,7 @@ public class TestCompactionArchiveConcurrentClose { private class WaitingHRegionFileSystem extends HRegionFileSystem { public WaitingHRegionFileSystem(final Configuration conf, final FileSystem fs, - final Path tableDir, final HRegionInfo regionInfo) { + final Path tableDir, final RegionInfo regionInfo) { super(conf, fs, tableDir, regionInfo); } http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java index 2b555a5..9c085e5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveIOException.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,38 +15,36 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.regionserver; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; - import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.FailedArchiveException; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -59,6 +57,8 @@ import org.junit.experimental.categories.Category; import org.junit.rules.TestName; import org.mockito.Mockito; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; + /** * Tests that archiving compacted files behaves correctly when encountering exceptions. */ @@ -93,11 +93,11 @@ public class TestCompactionArchiveIOException { byte[] col = Bytes.toBytes("c"); byte[] val = Bytes.toBytes("val"); - final TableName tableName = TableName.valueOf(name.getMethodName()); - HTableDescriptor htd = new HTableDescriptor(tableName); - htd.addFamily(new HColumnDescriptor(fam)); - HRegionInfo info = new HRegionInfo(tableName, null, null, false); - final HRegion region = initHRegion(htd, info); + TableName tableName = TableName.valueOf(name.getMethodName()); + TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) + .addColumnFamily(ColumnFamilyDescriptorBuilder.of(fam)).build(); + RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); + HRegion region = initHRegion(htd, info); RegionServerServices rss = mock(RegionServerServices.class); List<HRegion> regions = new ArrayList<>(); regions.add(region); @@ -172,29 +172,25 @@ public class TestCompactionArchiveIOException { } } - private HRegion initHRegion(HTableDescriptor htd, HRegionInfo info) - throws IOException { + private HRegion initHRegion(TableDescriptor htd, RegionInfo info) throws IOException { Configuration conf = testUtil.getConfiguration(); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); Path tableDir = FSUtils.getTableDir(testDir, htd.getTableName()); Path regionDir = new Path(tableDir, info.getEncodedName()); Path storeDir = new Path(regionDir, htd.getColumnFamilies()[0].getNameAsString()); - FileSystem errFS = spy(testUtil.getTestFileSystem()); // Prior to HBASE-16964, when an exception is thrown archiving any compacted file, // none of the other files are cleared from the compactedfiles list. // Simulate this condition with a dummy file - doThrow(new IOException("Error for test")) - .when(errFS).rename(eq(new Path(storeDir, ERROR_FILE)), any()); + doThrow(new IOException("Error for test")).when(errFS) + .rename(eq(new Path(storeDir, ERROR_FILE)), any()); HRegionFileSystem fs = new HRegionFileSystem(conf, errFS, tableDir, info); final Configuration walConf = new Configuration(conf); FSUtils.setRootDir(walConf, tableDir); final WALFactory wals = new WALFactory(walConf, null, "log_" + info.getEncodedName()); - HRegion region = - new HRegion(fs, wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), - conf, htd, null); + HRegion region = new HRegion(fs, wals.getWAL(info), conf, htd, null); region.initialize(); http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java index 7772f7b..b89fb0e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java @@ -30,7 +30,6 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CategoryBasedTimeout; @@ -39,18 +38,19 @@ import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueTestUtil; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; @@ -68,6 +68,7 @@ import org.junit.rules.TestName; import org.junit.rules.TestRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.base.Joiner; import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; @@ -451,22 +452,16 @@ public class TestDefaultMemStore { final MultiVersionConcurrencyControl mvcc; final MemStore memstore; - final AtomicLong startSeqNum; AtomicReference<Throwable> caughtException; - public ReadOwnWritesTester(int id, - MemStore memstore, - MultiVersionConcurrencyControl mvcc, - AtomicReference<Throwable> caughtException, - AtomicLong startSeqNum) - { + public ReadOwnWritesTester(int id, MemStore memstore, MultiVersionConcurrencyControl mvcc, + AtomicReference<Throwable> caughtException) { this.mvcc = mvcc; this.memstore = memstore; this.caughtException = caughtException; row = Bytes.toBytes(id); - this.startSeqNum = startSeqNum; } @Override @@ -505,14 +500,13 @@ public class TestDefaultMemStore { @Test public void testReadOwnWritesUnderConcurrency() throws Throwable { - int NUM_THREADS = 8; ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS]; AtomicReference<Throwable> caught = new AtomicReference<>(); for (int i = 0; i < NUM_THREADS; i++) { - threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught, this.startSeqNum); + threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught); threads[i].start(); } @@ -921,7 +915,8 @@ public class TestDefaultMemStore { EnvironmentEdgeManager.injectEdge(edge); HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf); String cf = "foo"; - HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor(cf)); + HRegion region = + hbaseUtility.createTestRegion("foobar", ColumnFamilyDescriptorBuilder.of(cf)); edge.setCurrentTimeMillis(1234); Put p = new Put(Bytes.toBytes("r")); @@ -950,20 +945,16 @@ public class TestDefaultMemStore { EnvironmentEdgeManager.injectEdge(edge); edge.setCurrentTimeMillis(1234); WALFactory wFactory = new WALFactory(conf, null, "1234"); - HRegion meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, + HRegion meta = HRegion.createHRegion(RegionInfoBuilder.FIRST_META_REGIONINFO, testDir, conf, FSTableDescriptors.createMetaTableDescriptor(conf), - wFactory.getMetaWAL(HRegionInfo.FIRST_META_REGIONINFO. - getEncodedNameAsBytes())); + wFactory.getWAL(RegionInfoBuilder.FIRST_META_REGIONINFO)); // parameterized tests add [#] suffix get rid of [ and ]. - HRegionInfo hri = - new HRegionInfo(TableName.valueOf(name.getMethodName().replaceAll("[\\[\\]]", "_")), - Bytes.toBytes("row_0200"), Bytes.toBytes("row_0300")); - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf( - name.getMethodName().replaceAll("[\\[\\]]", "_"))); - desc.addFamily(new HColumnDescriptor("foo".getBytes())); - HRegion r = - HRegion.createHRegion(hri, testDir, conf, desc, - wFactory.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace())); + TableDescriptor desc = TableDescriptorBuilder + .newBuilder(TableName.valueOf(name.getMethodName().replaceAll("[\\[\\]]", "_"))) + .addColumnFamily(ColumnFamilyDescriptorBuilder.of("foo")).build(); + RegionInfo hri = RegionInfoBuilder.newBuilder(desc.getTableName()) + .setStartKey(Bytes.toBytes("row_0200")).setEndKey(Bytes.toBytes("row_0300")).build(); + HRegion r = HRegion.createHRegion(hri, testDir, conf, desc, wFactory.getWAL(hri)); addRegionToMETA(meta, r); edge.setCurrentTimeMillis(1234 + 100); StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java index 32e3856..9845d8a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java @@ -162,8 +162,7 @@ public class TestHMobStore { final Configuration walConf = new Configuration(conf); FSUtils.setRootDir(walConf, basedir); final WALFactory wals = new WALFactory(walConf, null, methodName); - region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes(), - info.getTable().getNamespace()), fs, conf, info, htd, null); + region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, htd, null); store = new HMobStore(region, hcd, conf); if(testStore) { init(conf, hcd); http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 5ddd4df..3c11b31 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -63,7 +63,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; - import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -110,6 +109,7 @@ import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Scan; @@ -180,8 +180,10 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; + import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; @@ -378,9 +380,8 @@ public class TestHRegion { final Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); final Configuration walConf = new Configuration(conf); FSUtils.setRootDir(walConf, logDir); - return (new WALFactory(walConf, - Collections.<WALActionsListener>singletonList(new MetricsWAL()), callingMethod)) - .getWAL(tableName.toBytes(), tableName.getNamespace()); + return new WALFactory(walConf, Collections.<WALActionsListener> singletonList(new MetricsWAL()), + callingMethod).getWAL(RegionInfoBuilder.newBuilder(tableName).build()); } @Test @@ -960,7 +961,7 @@ public class TestHRegion { final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); FSUtils.setRootDir(walConf, logDir); final WALFactory wals = new WALFactory(walConf, null, method); - final WAL wal = wals.getWAL(tableName.getName(), tableName.getNamespace()); + final WAL wal = wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build()); this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, Durability.USE_DEFAULT, wal, family); @@ -4690,7 +4691,7 @@ public class TestHRegion { // deal with classes which have a field of an inner class. See discussions in HBASE-15536. walConf.set(WALFactory.WAL_PROVIDER, "filesystem"); final WALFactory wals = new WALFactory(walConf, null, UUID.randomUUID().toString()); - final WAL wal = spy(wals.getWAL(tableName.getName(), tableName.getNamespace())); + final WAL wal = spy(wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build())); this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, tableDurability, wal, new byte[][] { family }); http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java index 7715231..3c25f6b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.regionserver; import static org.apache.hadoop.hbase.regionserver.TestHRegion.assertGet; @@ -27,8 +26,8 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -43,7 +42,6 @@ import java.util.Map; import java.util.Objects; import java.util.Random; import java.util.UUID; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; @@ -51,7 +49,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -59,6 +56,8 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.executor.ExecutorService; @@ -89,8 +88,10 @@ import org.junit.rules.TestName; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; + import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor; @@ -132,7 +133,7 @@ public class TestHRegionReplayEvents { private TableDescriptor htd; private long time; private RegionServerServices rss; - private HRegionInfo primaryHri, secondaryHri; + private RegionInfo primaryHri, secondaryHri; private HRegion primaryRegion, secondaryRegion; private WALFactory wals; private WAL walPrimary, walSecondary; @@ -156,18 +157,14 @@ public class TestHRegionReplayEvents { time = System.currentTimeMillis(); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); - primaryHri = new HRegionInfo(htd.getTableName(), - HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, - false, time, 0); - secondaryHri = new HRegionInfo(htd.getTableName(), - HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, - false, time, 1); + primaryHri = + RegionInfoBuilder.newBuilder(htd.getTableName()).setRegionId(time).setReplicaId(0).build(); + secondaryHri = + RegionInfoBuilder.newBuilder(htd.getTableName()).setRegionId(time).setReplicaId(1).build(); wals = TestHRegion.createWALFactory(CONF, rootDir); - walPrimary = wals.getWAL(primaryHri.getEncodedNameAsBytes(), - primaryHri.getTable().getNamespace()); - walSecondary = wals.getWAL(secondaryHri.getEncodedNameAsBytes(), - secondaryHri.getTable().getNamespace()); + walPrimary = wals.getWAL(primaryHri); + walSecondary = wals.getWAL(secondaryHri); rss = mock(RegionServerServices.class); when(rss.getServerName()).thenReturn(ServerName.valueOf("foo", 1, 1)); @@ -1150,8 +1147,8 @@ public class TestHRegionReplayEvents { // test for region open and close secondaryRegion = HRegion.openHRegion(secondaryHri, htd, walSecondary, CONF, rss, null); - verify(walSecondary, times(0)).append((HRegionInfo)any(), - (WALKeyImpl)any(), (WALEdit)any(), anyBoolean()); + verify(walSecondary, times(0)).append(any(RegionInfo.class), any(WALKeyImpl.class), + any(WALEdit.class), anyBoolean()); // test for replay prepare flush putDataByReplay(secondaryRegion, 0, 10, cq, families); @@ -1166,12 +1163,12 @@ public class TestHRegionReplayEvents { primaryRegion.getRegionInfo().getRegionName())) .build()); - verify(walSecondary, times(0)).append((HRegionInfo)any(), - (WALKeyImpl)any(), (WALEdit)any(), anyBoolean()); + verify(walSecondary, times(0)).append(any(RegionInfo.class), any(WALKeyImpl.class), + any(WALEdit.class), anyBoolean()); secondaryRegion.close(); - verify(walSecondary, times(0)).append((HRegionInfo)any(), - (WALKeyImpl)any(), (WALEdit)any(), anyBoolean()); + verify(walSecondary, times(0)).append(any(RegionInfo.class), any(WALKeyImpl.class), + any(WALEdit.class), anyBoolean()); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java index e28e484..b8d3ec7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java @@ -21,7 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -47,7 +47,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -65,7 +64,6 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MemoryCompactionPolicy; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -73,6 +71,8 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; @@ -112,6 +112,7 @@ import org.junit.rules.TestName; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.collect.Lists; /** @@ -205,12 +206,12 @@ public class TestHStore { fs.delete(logdir, true); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, MemStoreLABImpl.CHUNK_SIZE_DEFAULT, 1, 0, null); - HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); + RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); Configuration walConf = new Configuration(conf); FSUtils.setRootDir(walConf, basedir); WALFactory wals = new WALFactory(walConf, null, methodName); - region = new HRegion(new HRegionFileSystem(conf, fs, tableDir, info), - wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), conf, htd, null); + region = new HRegion(new HRegionFileSystem(conf, fs, tableDir, info), wals.getWAL(info), conf, + htd, null); } private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder, @@ -1006,7 +1007,6 @@ public class TestHStore { assertEquals(0, this.store.getStorefilesCount()); } - @SuppressWarnings("unchecked") @Test public void testRefreshStoreFilesNotChanged() throws IOException { init(name.getMethodName()); http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java index 416b194..35c744f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java @@ -26,31 +26,33 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.wal.WALFactory; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.StoppableImplementation; +import org.apache.hadoop.hbase.wal.WALFactory; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -73,20 +75,20 @@ public class TestStoreFileRefresherChore { FSUtils.setRootDir(TEST_UTIL.getConfiguration(), testDir); } - private HTableDescriptor getTableDesc(TableName tableName, byte[]... families) { - HTableDescriptor htd = new HTableDescriptor(tableName); - for (byte[] family : families) { - HColumnDescriptor hcd = new HColumnDescriptor(family); - // Set default to be three versions. - hcd.setMaxVersions(Integer.MAX_VALUE); - htd.addFamily(hcd); - } - return htd; + private TableDescriptor getTableDesc(TableName tableName, int regionReplication, + byte[]... families) { + TableDescriptorBuilder builder = + TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(regionReplication); + Arrays.stream(families).map(family -> ColumnFamilyDescriptorBuilder.newBuilder(family) + .setMaxVersions(Integer.MAX_VALUE).build()).forEachOrdered(builder::addColumnFamily); + return builder.build(); } static class FailingHRegionFileSystem extends HRegionFileSystem { boolean fail = false; - FailingHRegionFileSystem(Configuration conf, FileSystem fs, Path tableDir, HRegionInfo regionInfo) { + + FailingHRegionFileSystem(Configuration conf, FileSystem fs, Path tableDir, + RegionInfo regionInfo) { super(conf, fs, tableDir, regionInfo); } @@ -99,21 +101,21 @@ public class TestStoreFileRefresherChore { } } - private HRegion initHRegion(HTableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId) + private HRegion initHRegion(TableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId) throws IOException { Configuration conf = TEST_UTIL.getConfiguration(); Path tableDir = FSUtils.getTableDir(testDir, htd.getTableName()); - HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false, 0, replicaId); - - HRegionFileSystem fs = new FailingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir, - info); + RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(startKey) + .setEndKey(stopKey).setRegionId(0L).setReplicaId(replicaId).build(); + HRegionFileSystem fs = + new FailingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir, info); final Configuration walConf = new Configuration(conf); FSUtils.setRootDir(walConf, tableDir); final WALFactory wals = new WALFactory(walConf, null, "log_" + replicaId); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); HRegion region = - new HRegion(fs, wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), + new HRegion(fs, wals.getWAL(info), conf, htd, null); region.initialize(); @@ -188,8 +190,7 @@ public class TestStoreFileRefresherChore { when(regionServer.getOnlineRegionsLocalContext()).thenReturn(regions); when(regionServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration()); - HTableDescriptor htd = getTableDesc(TableName.valueOf(name.getMethodName()), families); - htd.setRegionReplication(2); + TableDescriptor htd = getTableDesc(TableName.valueOf(name.getMethodName()), 2, families); HRegion primary = initHRegion(htd, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, 0); HRegion replica1 = initHRegion(htd, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, 1); regions.add(primary); http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALMonotonicallyIncreasingSeqId.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALMonotonicallyIncreasingSeqId.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALMonotonicallyIncreasingSeqId.java index 99db208..a5148b3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALMonotonicallyIncreasingSeqId.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALMonotonicallyIncreasingSeqId.java @@ -119,7 +119,7 @@ public class TestWALMonotonicallyIncreasingSeqId { wals = new WALFactory(walConf, null, "log_" + replicaId); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); HRegion region = HRegion.createHRegion(info, TEST_UTIL.getDefaultRootDirPath(), conf, htd, - wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace())); + wals.getWAL(info)); return region; } http://git-wip-us.apache.org/repos/asf/hbase/blob/814d08a2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java index 264b4c8..37c3b37 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.IOException; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -160,12 +159,12 @@ public abstract class AbstractTestLogRolling { /** * Tests that log rolling doesn't hang when no data is written. */ - @Test(timeout=120000) + @Test(timeout = 120000) public void testLogRollOnNothingWritten() throws Exception { final Configuration conf = TEST_UTIL.getConfiguration(); - final WALFactory wals = new WALFactory(conf, null, - ServerName.valueOf("test.com",8080, 1).toString()); - final WAL newLog = wals.getWAL(new byte[]{}, null); + final WALFactory wals = + new WALFactory(conf, null, ServerName.valueOf("test.com", 8080, 1).toString()); + final WAL newLog = wals.getWAL(null); try { // Now roll the log before we write anything. newLog.rollWriter(true);