Author: todd Date: Fri Dec 16 18:36:24 2011 New Revision: 1215241 URL: http://svn.apache.org/viewvc?rev=1215241&view=rev Log: HDFS-2684. Fix up some failing unit tests on HA branch. Contributed by Todd Lipcon.
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1215241&r1=1215240&r2=1215241&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt Fri Dec 16 18:36:24 2011 @@ -61,3 +61,5 @@ HDFS-2689. HA: BookKeeperEditLogInputStr HDFS-2602. NN should log newly-allocated blocks without losing BlockInfo (atm) HDFS-2667. Fix transition from active to standby (todd) + +HDFS-2684. Fix up some failing unit tests on HA branch (todd) Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java?rev=1215241&r1=1215240&r2=1215241&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java Fri Dec 16 18:36:24 2011 @@ -325,6 +325,8 @@ class BPOfferService { } else { bpRegistration = reg; } + + dn.bpRegistrationSucceeded(bpRegistration, getBlockPoolId()); } /** Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java?rev=1215241&r1=1215240&r2=1215241&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java Fri Dec 16 18:36:24 2011 @@ -199,6 +199,11 @@ public class BackupNode extends NameNode checkpointManager.interrupt(); checkpointManager = null; } + + // Abort current log segment - otherwise the NN shutdown code + // will close it gracefully, which is incorrect. + getFSImage().getEditLog().abortCurrentLogSegment(); + // Stop name-node threads super.stop(); } Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1215241&r1=1215240&r2=1215241&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Fri Dec 16 18:36:24 2011 @@ -916,6 +916,7 @@ public class FSEditLog { if (editLogStream != null) { editLogStream.abort(); editLogStream = null; + state = State.BETWEEN_LOG_SEGMENTS; } } catch (IOException e) { LOG.warn("All journals failed to abort", e); Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1215241&r1=1215240&r2=1215241&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Dec 16 18:36:24 2011 @@ -495,7 +495,7 @@ public class FSNamesystem implements Nam try { FSEditLog editLog = dir.fsImage.getEditLog(); - if (!editLog.isSegmentOpen()) { + if (!editLog.isOpenForWrite()) { // During startup, we're already open for write during initialization. // TODO(HA): consider adding a startup state? editLog.initJournalsForWrite(); @@ -2774,7 +2774,7 @@ public class FSNamesystem implements Nam DatanodeCommand[] cmds = blockManager.getDatanodeManager().handleHeartbeat( nodeReg, blockPoolId, capacity, dfsUsed, remaining, blockPoolUsed, xceiverCount, maxTransfer, failedVolumes); - if (cmds == null) { + if (cmds == null || cmds.length == 0) { DatanodeCommand cmd = upgradeManager.getBroadcastCommand(); if (cmd != null) { cmds = new DatanodeCommand[] {cmd}; Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java?rev=1215241&r1=1215240&r2=1215241&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java Fri Dec 16 18:36:24 2011 @@ -52,7 +52,7 @@ import static org.junit.Assert.*; */ public class TestDFSUpgrade { - private static final int EXPECTED_TXID = 33; + private static final int EXPECTED_TXID = 49; private static final Log LOG = LogFactory.getLog(TestDFSUpgrade.class.getName()); private Configuration conf; private int testCounter = 0; Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java?rev=1215241&r1=1215240&r2=1215241&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java Fri Dec 16 18:36:24 2011 @@ -302,7 +302,7 @@ public class TestDataTransferProtocol ex testWrite(firstBlock, BlockConstructionStage.PIPELINE_SETUP_CREATE, 0L, "Cannot create a RBW block", true); // test PIPELINE_SETUP_APPEND on an existing block - newGS = newBlock.getGenerationStamp() + 1; + newGS = firstBlock.getGenerationStamp() + 1; testWrite(firstBlock, BlockConstructionStage.PIPELINE_SETUP_APPEND, newGS, "Cannot append to a RBW replica", true); // test PIPELINE_SETUP_APPEND on an existing block Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java?rev=1215241&r1=1215240&r2=1215241&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java Fri Dec 16 18:36:24 2011 @@ -110,7 +110,7 @@ public class TestHeartbeatHandling exten cmds = NameNodeAdapter.sendHeartBeat(nodeReg, dd, namesystem) .getCommands(); - assertEquals(null, cmds); + assertEquals(0, cmds.length); } } finally { namesystem.writeUnlock(); Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1215241&r1=1215240&r2=1215241&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Fri Dec 16 18:36:24 2011 @@ -240,9 +240,9 @@ public class TestBackupNode extends Test } void testCheckpoint(StartupOption op) throws Exception { - Path file1 = new Path("checkpoint.dat"); - Path file2 = new Path("checkpoint2.dat"); - Path file3 = new Path("backup.dat"); + Path file1 = new Path("/checkpoint.dat"); + Path file2 = new Path("/checkpoint2.dat"); + Path file3 = new Path("/backup.dat"); Configuration conf = new HdfsConfiguration(); short replication = (short)conf.getInt("dfs.replication", 3); @@ -341,11 +341,13 @@ public class TestBackupNode extends Test TestCheckpoint.checkFile(fileSys, file3, replication); // should also be on BN right away assertTrue("file3 does not exist on BackupNode", - op != StartupOption.BACKUP || bnFS.exists(file3)); + op != StartupOption.BACKUP || + backup.getNamesystem().getFileInfo( + file3.toUri().getPath(), false) != null); } catch(IOException e) { LOG.error("Error in TestBackupNode:", e); - assertTrue(e.getLocalizedMessage(), false); + throw new AssertionError(e); } finally { if(backup != null) backup.stop(); if(fileSys != null) fileSys.close(); Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1215241&r1=1215240&r2=1215241&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Fri Dec 16 18:36:24 2011 @@ -923,10 +923,12 @@ public class TestCheckpoint extends Test throw new IOException(e); } + final int EXPECTED_TXNS_FIRST_SEG = 12; + // the following steps should have happened: - // edits_inprogress_1 -> edits_1-8 (finalized) - // fsimage_8 created - // edits_inprogress_9 created + // edits_inprogress_1 -> edits_1-12 (finalized) + // fsimage_12 created + // edits_inprogress_13 created // for(URI uri : editsDirs) { File ed = new File(uri.getPath()); @@ -938,19 +940,21 @@ public class TestCheckpoint extends Test NNStorage.getInProgressEditsFileName(1)); assertFalse(originalEdits.exists()); File finalizedEdits = new File(curDir, - NNStorage.getFinalizedEditsFileName(1,8)); - assertTrue(finalizedEdits.exists()); + NNStorage.getFinalizedEditsFileName(1, EXPECTED_TXNS_FIRST_SEG)); + GenericTestUtils.assertExists(finalizedEdits); assertTrue(finalizedEdits.length() > Integer.SIZE/Byte.SIZE); - assertTrue(new File(ed, "current/" - + NNStorage.getInProgressEditsFileName(9)).exists()); + GenericTestUtils.assertExists(new File(ed, "current/" + + NNStorage.getInProgressEditsFileName( + EXPECTED_TXNS_FIRST_SEG + 1))); } Collection<URI> imageDirs = cluster.getNameDirs(0); for (URI uri : imageDirs) { File imageDir = new File(uri.getPath()); File savedImage = new File(imageDir, "current/" - + NNStorage.getImageFileName(8)); + + NNStorage.getImageFileName( + EXPECTED_TXNS_FIRST_SEG)); assertTrue("Should have saved image at " + savedImage, savedImage.exists()); }