Author: todd
Date: Fri Dec 30 00:30:27 2011
New Revision: 1225709
URL: http://svn.apache.org/viewvc?rev=1225709&view=rev
Log:
HDFS-2692. Fix bugs related to failover from/into safe mode. Contributed by
Todd Lipcon.
Added:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1225709&r1=1225708&r2=1225709&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
Fri Dec 30 00:30:27 2011
@@ -77,3 +77,5 @@ HDFS-2693. Fix synchronization issues ar
HDFS-1972. Fencing mechanism for block invalidations and replications (todd)
HDFS-2714. Fix test cases which use standalone FSNamesystems (todd)
+
+HDFS-2692. Fix bugs related to failover from/into safe mode. (todd)
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1225709&r1=1225708&r2=1225709&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Fri Dec 30 00:30:27 2011
@@ -2481,7 +2481,10 @@ assert storedBlock.findDatanode(dn) < 0
public void removeBlock(Block block) {
assert namesystem.hasWriteLock();
- block.setNumBytes(BlockCommand.NO_ACK);
+ // TODO(HA): the following causes some problems for HA:
+ // the SBN doesn't get block deletions until the next
+ // BR...
+ // block.setNumBytes(BlockCommand.NO_ACK);
addToInvalidates(block);
corruptReplicas.removeFromCorruptReplicasMap(block);
blocksMap.removeBlock(block);
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java?rev=1225709&r1=1225708&r2=1225709&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
Fri Dec 30 00:30:27 2011
@@ -243,11 +243,16 @@ class Checkpointer extends Daemon {
long txid = bnImage.getLastAppliedTxId();
- backupNode.namesystem.dir.setReady();
- backupNode.namesystem.setBlockTotal();
-
- bnImage.saveFSImageInAllDirs(backupNode.getNamesystem(), txid);
- bnStorage.writeAll();
+ backupNode.namesystem.writeLock();
+ try {
+ backupNode.namesystem.dir.setReady();
+ backupNode.namesystem.setBlockTotal();
+
+ bnImage.saveFSImageInAllDirs(backupNode.getNamesystem(), txid);
+ bnStorage.writeAll();
+ } finally {
+ backupNode.namesystem.writeUnlock();
+ }
if(cpCmd.needToReturnImage()) {
TransferFsImage.uploadImageFromStorage(
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1225709&r1=1225708&r2=1225709&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
Fri Dec 30 00:30:27 2011
@@ -65,6 +65,7 @@ import com.google.common.base.Joiner;
@InterfaceStability.Evolving
public class FSEditLogLoader {
private final FSNamesystem fsNamesys;
+ private long maxGenStamp = 0;
public FSEditLogLoader(FSNamesystem fsNamesys) {
this.fsNamesys = fsNamesys;
@@ -78,14 +79,19 @@ public class FSEditLogLoader {
int loadFSEdits(EditLogInputStream edits, long expectedStartingTxId)
throws IOException {
long startTime = now();
- int numEdits = loadFSEdits(edits, true, expectedStartingTxId);
- FSImage.LOG.info("Edits file " + edits.getName()
- + " of size " + edits.length() + " edits # " + numEdits
- + " loaded in " + (now()-startTime)/1000 + " seconds.");
- return numEdits;
+ fsNamesys.writeLock();
+ try {
+ int numEdits = loadFSEdits(edits, true, expectedStartingTxId);
+ FSImage.LOG.info("Edits file " + edits.getName()
+ + " of size " + edits.length() + " edits # " + numEdits
+ + " loaded in " + (now()-startTime)/1000 + " seconds.");
+ return numEdits;
+ } finally {
+ fsNamesys.writeUnlock();
+ }
}
- int loadFSEdits(EditLogInputStream edits, boolean closeOnExit,
+ private int loadFSEdits(EditLogInputStream edits, boolean closeOnExit,
long expectedStartingTxId)
throws IOException {
int numEdits = 0;
@@ -95,6 +101,13 @@ public class FSEditLogLoader {
numEdits = loadEditRecords(logVersion, edits, false,
expectedStartingTxId);
} finally {
+ fsNamesys.setBlockTotal();
+ // Delay the notification of genstamp updates until after
+ // setBlockTotal() above. Otherwise, we will mark blocks
+ // as "safe" before they've been incorporated in the expected
+ // totalBlocks and threshold for SafeMode -- triggering an
+ // assertion failure and/or exiting safemode too early!
+ fsNamesys.notifyGenStampUpdate(maxGenStamp);
if(closeOnExit) {
edits.close();
}
@@ -485,9 +498,9 @@ public class FSEditLogLoader {
}
}
- if (addCloseOp.blocks.length > 0) {
- fsNamesys.notifyGenStampUpdate(
- addCloseOp.blocks[addCloseOp.blocks.length -
1].getGenerationStamp());
+ // Record the max genstamp seen
+ for (Block b : addCloseOp.blocks) {
+ maxGenStamp = Math.max(maxGenStamp, b.getGenerationStamp());
}
}
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1225709&r1=1225708&r2=1225709&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Fri Dec 30 00:30:27 2011
@@ -418,6 +418,7 @@ public class FSNamesystem implements Nam
startOpt = StartupOption.REGULAR;
}
boolean success = false;
+ writeLock();
try {
// We shouldn't be calling saveNamespace if we've come up in standby
state.
if (fsImage.recoverTransitionRead(startOpt, this) && !haEnabled) {
@@ -434,6 +435,7 @@ public class FSNamesystem implements Nam
if (!success) {
fsImage.close();
}
+ writeUnlock();
}
dir.imageLoadComplete();
}
@@ -3244,9 +3246,7 @@ public class FSNamesystem implements Nam
* @return true if in safe mode
*/
private synchronized boolean isOn() {
- assert isConsistent() : " SafeMode: Inconsistent filesystem state: "
- + "Total num of blocks, active blocks, or "
- + "total safe blocks don't match.";
+ doConsistencyCheck();
return this.reached >= 0;
}
@@ -3362,6 +3362,9 @@ public class FSNamesystem implements Nam
* Check and trigger safe mode if needed.
*/
private void checkMode() {
+ // Have to have write-lock since leaving safemode initializes
+ // repl queues, which requires write lock
+ assert hasWriteLock();
if (needEnter()) {
enter();
// check if we are ready to initialize replication queues
@@ -3541,16 +3544,26 @@ public class FSNamesystem implements Nam
/**
* Checks consistency of the class state.
- * This is costly and currently called only in assert.
- * @throws IOException
+ * This is costly so only runs if asserts are enabled.
*/
- private boolean isConsistent() {
+ private void doConsistencyCheck() {
+ boolean assertsOn = false;
+ assert assertsOn = true; // set to true if asserts are on
+ if (!assertsOn) return;
+
+
+ int activeBlocks = blockManager.getActiveBlockCount();
if (blockTotal == -1 && blockSafe == -1) {
- return true; // manual safe mode
+ return; // manual safe mode
+ }
+ if ((blockTotal != activeBlocks) &&
+ !(blockSafe >= 0 && blockSafe <= blockTotal)) {
+ throw new AssertionError(
+ " SafeMode: Inconsistent filesystem state: "
+ + "SafeMode data: blockTotal=" + blockTotal
+ + " blockSafe=" + blockSafe + "; "
+ + "BlockManager data: active=" + activeBlocks);
}
- int activeBlocks = blockManager.getActiveBlockCount();
- return (blockTotal == activeBlocks) ||
- (blockSafe >= 0 && blockSafe <= blockTotal);
}
}
@@ -3663,7 +3676,7 @@ public class FSNamesystem implements Nam
/**
* Set the total number of blocks in the system.
*/
- void setBlockTotal() {
+ public void setBlockTotal() {
// safeMode is volatile, and may be set to null at any time
SafeModeInfo safeMode = this.safeMode;
if (safeMode == null)
@@ -4822,10 +4835,15 @@ public class FSNamesystem implements Nam
}
public void notifyGenStampUpdate(long gs) {
- LOG.info("=> notified of genstamp update for: " + gs);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Generation stamp " + gs + " has been reached. " +
+ "Processing pending messages from DataNodes...");
+ }
DataNodeMessage msg = pendingDatanodeMessages.take(gs);
while (msg != null) {
- LOG.info("processing message: " + msg);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing previously pending message: " + msg);
+ }
try {
switch (msg.getType()) {
case BLOCK_RECEIVED_DELETE:
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java?rev=1225709&r1=1225708&r2=1225709&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
Fri Dec 30 00:30:27 2011
@@ -49,6 +49,8 @@ public class EditLogTailer {
private final FSImage image;
private final FSEditLog editLog;
+ private volatile Throwable lastError = null;
+
public EditLogTailer(FSNamesystem namesystem) {
this.tailerThread = new EditLogTailerThread();
this.namesystem = namesystem;
@@ -81,6 +83,11 @@ public class EditLogTailer {
tailerThread.interrupt();
}
+ @VisibleForTesting
+ public Throwable getLastError() {
+ return lastError;
+ }
+
public void catchupDuringFailover() throws IOException {
Preconditions.checkState(tailerThread == null ||
!tailerThread.isAlive(),
@@ -146,12 +153,19 @@ public class EditLogTailer {
try {
doTailEdits();
} catch (IOException e) {
+ if (e.getCause() instanceof RuntimeException) {
+ throw (RuntimeException)e.getCause();
+ } else if (e.getCause() instanceof Error) {
+ throw (Error)e.getCause();
+ }
+
// Will try again
LOG.info("Got error, will try again.", e);
}
} catch (Throwable t) {
// TODO(HA): What should we do in this case? Shutdown the standby NN?
LOG.error("Edit log tailer received throwable", t);
+ lastError = t;
}
try {
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1225709&r1=1225708&r2=1225709&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
Fri Dec 30 00:30:27 2011
@@ -1452,7 +1452,8 @@ public class MiniDFSCluster {
sizes = NameNodeAdapter.getStats(nameNode.getNamesystem());
boolean isUp = false;
synchronized (this) {
- isUp = ((!nameNode.isInSafeMode() || !waitSafeMode) && sizes[0] != 0);
+ isUp = ((!nameNode.isInSafeMode() || !waitSafeMode) &&
+ sizes[ClientProtocol.GET_STATS_CAPACITY_IDX] != 0);
}
return isUp;
}
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java?rev=1225709&r1=1225708&r2=1225709&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
(original)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
Fri Dec 30 00:30:27 2011
@@ -410,7 +410,7 @@ public class TestDNFencing {
return count;
}
- private void waitForDNDeletions(final MiniDFSCluster cluster)
+ static void waitForDNDeletions(final MiniDFSCluster cluster)
throws TimeoutException, InterruptedException {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
@@ -426,7 +426,7 @@ public class TestDNFencing {
}
- private void waitForNNToIssueDeletions(final NameNode nn)
+ static void waitForNNToIssueDeletions(final NameNode nn)
throws Exception {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
Added:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java?rev=1225709&view=auto
==============================================================================
---
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
(added)
+++
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
Fri Dec 30 00:30:27 2011
@@ -0,0 +1,334 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.ha;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.TestDFSClientFailover;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests that exercise safemode in an HA cluster.
+ */
+public class TestHASafeMode {
+ private static final Log LOG = LogFactory.getLog(TestHASafeMode.class);
+ private static final int BLOCK_SIZE = 1024;
+ private NameNode nn0;
+ private NameNode nn1;
+ private FileSystem fs;
+ private MiniDFSCluster cluster;
+
+ @Before
+ public void setupCluster() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
+ .numDataNodes(3)
+ .waitSafeMode(false)
+ .build();
+ cluster.waitActive();
+
+ nn0 = cluster.getNameNode(0);
+ nn1 = cluster.getNameNode(1);
+ fs = TestDFSClientFailover.configureFailoverFs(cluster, conf);
+
+ cluster.transitionToActive(0);
+ }
+
+ @After
+ public void shutdownCluster() throws IOException {
+ if (cluster != null) {
+ assertNull(nn1.getNamesystem().getEditLogTailer().getLastError());
+ cluster.shutdown();
+ }
+ }
+
+ private void restartStandby() throws IOException {
+ cluster.shutdownNameNode(1);
+ // Set the safemode extension to be lengthy, so that the tests
+ // can check the safemode message after the safemode conditions
+ // have been achieved, without being racy.
+ cluster.getConfiguration(1).setInt(
+ DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 30000);
+ cluster.restartNameNode(1);
+ nn1 = cluster.getNameNode(1);
+ nn1.getNamesystem().getEditLogTailer().setSleepTime(250);
+ nn1.getNamesystem().getEditLogTailer().interrupt();
+ }
+
+ /**
+ * Tests the case where, while a standby is down, more blocks are
+ * added to the namespace, but not rolled. So, when it starts up,
+ * it receives notification about the new blocks during
+ * the safemode extension period.
+ */
+ @Test
+ public void testBlocksAddedBeforeStandbyRestart() throws Exception {
+ banner("Starting with NN0 active and NN1 standby, creating some blocks");
+ DFSTestUtil.createFile(fs, new Path("/test"), 3*BLOCK_SIZE, (short) 3, 1L);
+ // Roll edit log so that, when the SBN restarts, it will load
+ // the namespace during startup.
+ nn0.getRpcServer().rollEditLog();
+
+ banner("Creating some blocks that won't be in the edit log");
+ DFSTestUtil.createFile(fs, new Path("/test2"), 5*BLOCK_SIZE, (short) 3,
1L);
+
+ banner("Restarting standby");
+ restartStandby();
+
+ // We expect it to be stuck in safemode (not the extension) because
+ // the block reports are delayed (since they include blocks
+ // from /test2 which are too-high genstamps.
+ String status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 0 needs additional 3 blocks to reach"));
+
+ banner("Waiting for standby to catch up to active namespace");
+ TestEditLogTailer.waitForStandbyToCatchUp(nn0, nn1);
+
+ status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 8 has reached the threshold 0.9990 of " +
+ "total blocks 8. Safe mode will be turned off automatically"));
+ }
+
+ /**
+ * Similar to {@link #testBlocksAddedBeforeStandbyRestart()} except that
+ * the new blocks are allocated after the SBN has restarted. So, the
+ * blocks were not present in the original block reports at startup
+ * but are reported separately by blockReceived calls.
+ */
+ @Test
+ public void testBlocksAddedWhileInSafeMode() throws Exception {
+ banner("Starting with NN0 active and NN1 standby, creating some blocks");
+ DFSTestUtil.createFile(fs, new Path("/test"), 3*BLOCK_SIZE, (short) 3, 1L);
+ // Roll edit log so that, when the SBN restarts, it will load
+ // the namespace during startup.
+ nn0.getRpcServer().rollEditLog();
+
+ banner("Restarting standby");
+ restartStandby();
+
+ String status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 3 has reached the threshold 0.9990 of " +
+ "total blocks 3. Safe mode will be turned off automatically"));
+
+ // Create a few blocks which will send blockReceived calls to the
+ // SBN.
+ banner("Creating some blocks while SBN is in safe mode");
+ DFSTestUtil.createFile(fs, new Path("/test2"), 5*BLOCK_SIZE, (short) 3,
1L);
+
+
+ banner("Waiting for standby to catch up to active namespace");
+ TestEditLogTailer.waitForStandbyToCatchUp(nn0, nn1);
+
+ status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 8 has reached the threshold 0.9990 of " +
+ "total blocks 8. Safe mode will be turned off automatically"));
+ }
+
+ /**
+ * Test for the following case proposed by ATM:
+ * 1. Both NNs are up, one is active. There are 100 blocks. Both are
+ * out of safemode.
+ * 2. 10 block deletions get processed by NN1. NN2 enqueues these DN messages
+ * until it next reads from a checkpointed edits file.
+ * 3. NN2 gets restarted. Its queues are lost.
+ * 4. NN2 comes up, reads from all the finalized edits files. Concludes there
+ * should still be 100 blocks.
+ * 5. NN2 receives a block report from all the DNs, which only accounts for
+ * 90 blocks. It doesn't leave safemode.
+ * 6. NN1 dies or is transitioned to standby.
+ * 7. NN2 is transitioned to active. It reads all the edits from NN1. It now
+ * knows there should only be 90 blocks, but it's still in safemode.
+ * 8. NN2 doesn't ever recheck whether it should leave safemode.
+ *
+ * This is essentially the inverse of {@link
#testBlocksAddedWhileStandbyShutdown()}
+ */
+ @Test
+ public void testBlocksRemovedBeforeStandbyRestart() throws Exception {
+ banner("Starting with NN0 active and NN1 standby, creating some blocks");
+ DFSTestUtil.createFile(fs, new Path("/test"), 5*BLOCK_SIZE, (short) 3, 1L);
+
+ // Roll edit log so that, when the SBN restarts, it will load
+ // the namespace during startup.
+ nn0.getRpcServer().rollEditLog();
+
+ // Delete those blocks again, so they won't get reported to the SBN
+ // once it starts up
+ banner("Removing the blocks without rolling the edit log");
+ fs.delete(new Path("/test"), true);
+ BlockManagerTestUtil.computeAllPendingWork(
+ nn0.getNamesystem().getBlockManager());
+ cluster.triggerHeartbeats();
+
+ banner("Restarting standby");
+ restartStandby();
+ String status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 0 needs additional 5 blocks to reach"));
+
+ banner("Waiting for standby to catch up to active namespace");
+ TestEditLogTailer.waitForStandbyToCatchUp(nn0, nn1);
+ status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 0 has reached the threshold 0.9990 of " +
+ "total blocks 0. Safe mode will be turned off automatically"));
+ }
+
+ /**
+ * Similar to {@link #testBlocksRemovedBeforeStandbyRestart()} except that
+ * the blocks are removed after the SBN has restarted. So, the
+ * blocks were present in the original block reports at startup
+ * but are deleted separately later by deletion reports.
+ */
+ @Test
+ public void testBlocksRemovedWhileInSafeMode() throws Exception {
+ banner("Starting with NN0 active and NN1 standby, creating some blocks");
+ DFSTestUtil.createFile(fs, new Path("/test"), 10*BLOCK_SIZE, (short) 3,
1L);
+
+ // Roll edit log so that, when the SBN restarts, it will load
+ // the namespace during startup.
+ nn0.getRpcServer().rollEditLog();
+
+ banner("Restarting standby");
+ restartStandby();
+
+ // It will initially have all of the blocks necessary.
+ String status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 10 has reached the threshold 0.9990 of " +
+ "total blocks 10. Safe mode will be turned off automatically"));
+
+ // Delete those blocks while the SBN is in safe mode - this
+ // should reduce it back below the threshold
+ banner("Removing the blocks without rolling the edit log");
+ fs.delete(new Path("/test"), true);
+ BlockManagerTestUtil.computeAllPendingWork(
+ nn0.getNamesystem().getBlockManager());
+
+ banner("Triggering deletions on DNs and Deletion Reports");
+ cluster.triggerHeartbeats();
+ TestDNFencing.waitForDNDeletions(cluster);
+ cluster.triggerDeletionReports();
+
+ status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 0 needs additional 10 blocks"));
+
+ banner("Waiting for standby to catch up to active namespace");
+ TestEditLogTailer.waitForStandbyToCatchUp(nn0, nn1);
+
+ status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 0 has reached the threshold 0.9990 of " +
+ "total blocks 0. Safe mode will be turned off automatically"));
+ }
+
+ /**
+ * Set up a namesystem with several edits, both deletions and
+ * additions, and failover to a new NN while that NN is in
+ * safemode. Ensure that it will exit safemode.
+ */
+ @Test
+ public void testComplexFailoverIntoSafemode() throws Exception {
+ banner("Starting with NN0 active and NN1 standby, creating some blocks");
+ DFSTestUtil.createFile(fs, new Path("/test"), 3*BLOCK_SIZE, (short) 3, 1L);
+ // Roll edit log so that, when the SBN restarts, it will load
+ // the namespace during startup and enter safemode.
+ nn0.getRpcServer().rollEditLog();
+
+ banner("Creating some blocks that won't be in the edit log");
+ DFSTestUtil.createFile(fs, new Path("/test2"), 5*BLOCK_SIZE, (short) 3,
1L);
+
+ banner("Deleting the original blocks");
+ fs.delete(new Path("/test"), true);
+
+ banner("Restarting standby");
+ restartStandby();
+
+ // We expect it to be stuck in safemode (not the extension) because
+ // the block reports are delayed (since they include blocks
+ // from /test2 which are too-high genstamps.
+ String status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 0 needs additional 3 blocks to reach"));
+
+ // Initiate a failover into it while it's in safemode
+ banner("Initiating a failover into NN1 in safemode");
+ NameNodeAdapter.abortEditLogs(nn0);
+ cluster.transitionToActive(1);
+
+ status = nn1.getNamesystem().getSafemode();
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON." +
+ "The reported blocks 5 has reached the threshold 0.9990 of " +
+ "total blocks 5. Safe mode will be turned off automatically"));
+ }
+
+ /**
+ * Print a big banner in the test log to make debug easier.
+ */
+ static void banner(String string) {
+ LOG.info("\n\n\n\n================================================\n" +
+ string + "\n" +
+ "==================================================\n\n");
+ }
+
+}
