Repository: hbase Updated Branches: refs/heads/0.98 06c4d8a8e -> bc594c079
HBASE-12631 Backport HBASE-12576 (Add metrics for rolling the HLog if there are too few DNs in the write pipeline) to 0.98 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bc594c07 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bc594c07 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bc594c07 Branch: refs/heads/0.98 Commit: bc594c0793b5cbeedccf7633d96e901626685ec9 Parents: 4c1a31f Author: Andrew Purtell <[email protected]> Authored: Wed Dec 10 17:23:06 2014 -0800 Committer: Andrew Purtell <[email protected]> Committed: Wed Dec 10 17:23:06 2014 -0800 ---------------------------------------------------------------------- .../regionserver/wal/MetricsWALSource.java | 9 ++++ .../regionserver/wal/MetricsWALSourceImpl.java | 21 ++++++-- .../regionserver/wal/MetricsWALSourceImpl.java | 19 ++++++- .../hadoop/hbase/regionserver/LogRoller.java | 2 +- .../hadoop/hbase/regionserver/wal/FSHLog.java | 16 +++--- .../hbase/regionserver/wal/MetricsWAL.java | 52 ++++++++++++++++++- .../regionserver/wal/WALActionsListener.java | 3 +- .../replication/regionserver/Replication.java | 2 +- .../hbase/mapreduce/TestImportExport.java | 2 +- .../hadoop/hbase/regionserver/wal/TestHLog.java | 2 +- .../regionserver/wal/TestLogRollPeriod.java | 2 +- .../hbase/regionserver/wal/TestLogRolling.java | 54 ++++++++++++++------ .../hbase/regionserver/wal/TestMetricsWAL.java | 44 ++++++++++++++++ .../wal/TestWALActionsListener.java | 2 +- .../TestReplicationHLogReaderManager.java | 2 +- 15 files changed, 195 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java index 1c59f65..b642a49 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java @@ -57,6 +57,11 @@ public interface MetricsWALSource extends BaseSource { String SLOW_APPEND_COUNT_DESC = "Number of appends that were slow."; String SYNC_TIME = "syncTime"; String SYNC_TIME_DESC = "The time it took to sync the HLog to HDFS."; + String ROLL_REQUESTED = "rollRequest"; + String ROLL_REQUESTED_DESC = "How many times a log roll has been requested total"; + String LOW_REPLICA_ROLL_REQUESTED = "lowReplicaRollRequest"; + String LOW_REPLICA_ROLL_REQUESTED_DESC = + "How many times a log roll was requested due to too few DN's in the write pipeline."; /** * Add the append size. @@ -83,4 +88,8 @@ public interface MetricsWALSource extends BaseSource { */ void incrementSyncTime(long time); + void incrementLogRollRequested(); + + void incrementLowReplicationLogRoll(); + } http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java index d1a318e..505e989 100644 --- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java @@ -22,7 +22,6 @@ import org.apache.hadoop.hbase.metrics.BaseSourceImpl; import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.lib.MetricMutableCounterLong; - /** * Class that transitions metrics from HLog's MetricsWAL into the metrics subsystem. * @@ -35,7 +34,8 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo private final MetricMutableCounterLong appendCount; private final MetricMutableCounterLong slowAppendCount; private final MetricHistogram syncTimeHisto; - + private final MetricMutableCounterLong logRollRequested; + private final MetricMutableCounterLong lowReplicationLogRollRequested; public MetricsWALSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); @@ -51,8 +51,13 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo appendTimeHisto = this.getMetricsRegistry().newHistogram(APPEND_TIME, APPEND_TIME_DESC); appendSizeHisto = this.getMetricsRegistry().newHistogram(APPEND_SIZE, APPEND_SIZE_DESC); appendCount = this.getMetricsRegistry().newCounter(APPEND_COUNT, APPEND_COUNT_DESC, 0l); - slowAppendCount = this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l); + slowAppendCount = + this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l); syncTimeHisto = this.getMetricsRegistry().newHistogram(SYNC_TIME, SYNC_TIME_DESC); + logRollRequested = + this.getMetricsRegistry().newCounter(ROLL_REQUESTED, ROLL_REQUESTED_DESC, 0L); + lowReplicationLogRollRequested = this.getMetricsRegistry() + .newCounter(LOW_REPLICA_ROLL_REQUESTED, LOW_REPLICA_ROLL_REQUESTED_DESC, 0L); } @Override @@ -79,4 +84,14 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo public void incrementSyncTime(long time) { syncTimeHisto.add(time); } + + @Override + public void incrementLogRollRequested() { + logRollRequested.incr(); + } + + @Override + public void incrementLowReplicationLogRoll() { + lowReplicationLogRollRequested.incr(); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java index ad8f24c..f6f12b8 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java @@ -37,6 +37,8 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo private final MetricHistogram syncTimeHisto; private final MutableCounterLong appendCount; private final MutableCounterLong slowAppendCount; + private final MutableCounterLong logRollRequested; + private final MutableCounterLong lowReplicationLogRollRequested; public MetricsWALSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); @@ -52,8 +54,13 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo appendTimeHisto = this.getMetricsRegistry().newHistogram(APPEND_TIME, APPEND_TIME_DESC); appendSizeHisto = this.getMetricsRegistry().newHistogram(APPEND_SIZE, APPEND_SIZE_DESC); appendCount = this.getMetricsRegistry().newCounter(APPEND_COUNT, APPEND_COUNT_DESC, 0l); - slowAppendCount = this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l); + slowAppendCount = + this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l); syncTimeHisto = this.getMetricsRegistry().newHistogram(SYNC_TIME, SYNC_TIME_DESC); + logRollRequested = + this.getMetricsRegistry().newCounter(ROLL_REQUESTED, ROLL_REQUESTED_DESC, 0L); + lowReplicationLogRollRequested = this.getMetricsRegistry() + .newCounter(LOW_REPLICA_ROLL_REQUESTED, LOW_REPLICA_ROLL_REQUESTED_DESC, 0L); } @Override @@ -80,4 +87,14 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo public void incrementSyncTime(long time) { syncTimeHisto.add(time); } + + @Override + public void incrementLogRollRequested() { + logRollRequested.incr(); + } + + @Override + public void incrementLowReplicationLogRoll() { + lowReplicationLogRollRequested.incr(); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java index 0e98143..45722e3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java @@ -141,7 +141,7 @@ class LogRoller extends HasThread implements WALActionsListener { } } - public void logRollRequested() { + public void logRollRequested(boolean lowReplicas) { synchronized (rollLog) { rollLog.set(true); rollLog.notifyAll(); http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index f96e645..b528871 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -36,7 +36,6 @@ import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; @@ -439,6 +438,7 @@ class FSHLog implements HLog, Syncable { coprocessorHost = new WALCoprocessorHost(this, conf); this.metrics = new MetricsWAL(); + registerWALActionsListener(metrics); } /** @@ -1258,16 +1258,16 @@ class FSHLog implements HLog, Syncable { asyncNotifier.setFlushedTxid(this.lastSyncedTxid); // 4. check and do logRoll if needed - boolean logRollNeeded = false; + boolean lowReplication = false; if (rollWriterLock.tryLock()) { try { - logRollNeeded = checkLowReplication(); + lowReplication = checkLowReplication(); } finally { rollWriterLock.unlock(); } try { - if (logRollNeeded || writer != null && writer.getLength() > logrollsize) { - requestLogRoll(); + if (lowReplication || writer != null && writer.getLength() > logrollsize) { + requestLogRoll(lowReplication); } } catch (IOException e) { LOG.warn("writer.getLength() failed,this failure won't block here"); @@ -1467,9 +1467,13 @@ class FSHLog implements HLog, Syncable { } private void requestLogRoll() { + requestLogRoll(false); + } + + private void requestLogRoll(boolean tooFewReplicas) { if (!this.listeners.isEmpty()) { for (WALActionsListener i: this.listeners) { - i.logRollRequested(); + i.logRollRequested(tooFewReplicas); } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java index cf4b7a6..ec1e89f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java @@ -19,25 +19,37 @@ package org.apache.hadoop.hbase.regionserver.wal; +import java.io.IOException; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.util.StringUtils; +import com.google.common.annotations.VisibleForTesting; + /** * Class used to push numbers about the WAL into the metrics subsystem. This will take a * single function call and turn it into multiple manipulations of the hadoop metrics system. */ @InterfaceAudience.Private -public class MetricsWAL { +public class MetricsWAL implements WALActionsListener { static final Log LOG = LogFactory.getLog(MetricsWAL.class); private final MetricsWALSource source; public MetricsWAL() { - source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); + this(CompatibilitySingletonFactory.getInstance(MetricsWALSource.class)); + } + + @VisibleForTesting + MetricsWAL(MetricsWALSource s) { + this.source = s; } public void finishSync(long time) { @@ -58,4 +70,40 @@ public class MetricsWAL { StringUtils.humanReadableInt(size))); } } + + @Override + public void logRollRequested(boolean underReplicated) { + source.incrementLogRollRequested(); + if (underReplicated) { + source.incrementLowReplicationLogRoll(); + } + } + + @Override + public void preLogRoll(Path oldPath, Path newPath) throws IOException { + } + + @Override + public void postLogRoll(Path oldPath, Path newPath) throws IOException { + } + + @Override + public void preLogArchive(Path oldPath, Path newPath) throws IOException { + } + + @Override + public void postLogArchive(Path oldPath, Path newPath) throws IOException { + } + + @Override + public void logCloseRequested() { + } + + @Override + public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) { + } + + @Override + public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) { + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java index 84552a7..dee042d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java @@ -64,8 +64,9 @@ public interface WALActionsListener { /** * A request was made that the WAL be rolled. + * @param tooFewReplicas roll requested because of too few replicas if true */ - void logRollRequested(); + void logRollRequested(boolean tooFewReplicas); /** * The WAL is about to close. http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java index 15ccffe..b665d70 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java @@ -298,7 +298,7 @@ public class Replication implements WALActionsListener, } @Override - public void logRollRequested() { + public void logRollRequested(boolean tooFewReplicas) { // Not interested } http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java index 3e6934d..80f6ac5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java @@ -705,7 +705,7 @@ public class TestImportExport { } @Override - public void logRollRequested() { + public void logRollRequested(boolean tooFewReplicas) { // Not interested in this method. } http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java index ce9a147..9bb03be 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java @@ -1222,7 +1222,7 @@ public class TestHLog { } @Override - public void logRollRequested() { + public void logRollRequested(boolean tooFewReplicas) { // TODO Auto-generated method stub } http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java index 30c3be8..a102103 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java @@ -149,7 +149,7 @@ public class TestLogRollPeriod { @Override public void postLogArchive(Path oldFile, Path newFile) {} @Override - public void logRollRequested() {} + public void logRollRequested(boolean tooFewReplicas) {} @Override public void logCloseRequested() {} @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java index 4e50519..d69d095 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java @@ -32,10 +32,10 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -63,12 +63,10 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.LeaseManager; -import org.apache.log4j.Level; + import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -92,18 +90,6 @@ public class TestLogRolling { private MiniHBaseCluster cluster; private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - // verbose logging on classes that are touched in these tests - { - ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); - ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); - ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")) - .getLogger().setLevel(Level.ALL); - ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); - ((Log4JLogger)HRegionServer.LOG).getLogger().setLevel(Level.ALL); - ((Log4JLogger)HRegion.LOG).getLogger().setLevel(Level.ALL); - ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL); - } - /** * constructor * @throws Exception @@ -341,6 +327,37 @@ public class TestLogRolling { server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName)); this.log = server.getWAL(); + final AtomicBoolean lowReplicationHookCalled = new AtomicBoolean(false); + + log.registerWALActionsListener(new WALActionsListener() { + @Override + public void logRollRequested(boolean lowReplication) { + if (lowReplication) { + lowReplicationHookCalled.lazySet(true); + } + } + + @Override + public void preLogRoll(Path oldPath, Path newPath) throws IOException { } + + @Override + public void postLogRoll(Path oldPath, Path newPath) throws IOException { } + + @Override + public void preLogArchive(Path oldPath, Path newPath) throws IOException { } + + @Override + public void postLogArchive(Path oldPath, Path newPath) throws IOException { } + + @Override + public void logCloseRequested() { } + + @Override + public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) { } + + @Override + public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) { } + }); assertTrue("Need HDFS-826 for this test", ((FSHLog) log).canGetCurReplicas()); // don't run this test without append support (HDFS-200 & HDFS-142) @@ -389,6 +406,9 @@ public class TestLogRolling { assertTrue("Missing datanode should've triggered a log roll", newFilenum > oldFilenum && newFilenum > curTime); + assertTrue("The log rolling hook should have been called with the low replication flag", + lowReplicationHookCalled.get()); + // write some more log data (this should use a new hdfs_out) writeData(table, 3); assertTrue("The log should not roll again.", @@ -461,7 +481,7 @@ public class TestLogRolling { @Override public void postLogArchive(Path oldFile, Path newFile) {} @Override - public void logRollRequested() {} + public void logRollRequested(boolean tooFewReplicas) {} @Override public void logCloseRequested() {} @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java new file mode 100644 index 0000000..719b80e --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java @@ -0,0 +1,44 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@Category(SmallTests.class) +public class TestMetricsWAL { + @Test + public void testLogRollRequested() throws Exception { + MetricsWALSource source = mock(MetricsWALSourceImpl.class); + MetricsWAL metricsWAL = new MetricsWAL(source); + metricsWAL.logRollRequested(false); + metricsWAL.logRollRequested(true); + + // Log roll was requested twice + verify(source, times(2)).incrementLogRollRequested(); + // One was because of low replication on the hlog. + verify(source, times(1)).incrementLowReplicationLogRoll(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java index 85b1dc3..4d50fbb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java @@ -152,7 +152,7 @@ public class TestWALActionsListener { } @Override - public void logRollRequested() { + public void logRollRequested(boolean tooFewReplicas) { // Not interested } http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java index cdaeb58..2fd16d0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java @@ -225,7 +225,7 @@ public class TestReplicationHLogReaderManager { public void postLogArchive(Path oldPath, Path newPath) throws IOException {} @Override - public void logRollRequested() {} + public void logRollRequested(boolean tooFewReplicas) {} @Override public void logCloseRequested() {}
