HDFS-6800. Support Datanode layout changes with rolling upgrade. (Contributed by James Thomas)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ae8178c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ae8178c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ae8178c Branch: refs/heads/HDFS-6581 Commit: 4ae8178c5626d188b137e3f806e56fd8661c4970 Parents: fa80ca4 Author: arp <a...@apache.org> Authored: Fri Aug 29 00:26:13 2014 -0700 Committer: arp <a...@apache.org> Committed: Fri Aug 29 00:26:13 2014 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java | 9 ++++++++- .../org/apache/hadoop/hdfs/server/datanode/DataNode.java | 5 ++--- .../hadoop/hdfs/server/namenode/NameNodeRpcServer.java | 1 + .../hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml | 2 +- 5 files changed, 15 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae8178c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8268b6b..957034b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -571,6 +571,9 @@ Release 2.6.0 - UNRELEASED HDFS-6902. FileWriter should be closed in finally block in BlockReceiver#receiveBlock() (Tsuyoshi OZAWA via Colin Patrick McCabe) + HDFS-6800. Support Datanode layout changes with rolling upgrade. + (James Thomas via Arpit Agarwal) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae8178c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java index 8e65dd0..88f858b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java @@ -255,7 +255,14 @@ public class BlockPoolSliceStorage extends Storage { */ private void doTransition(DataNode datanode, StorageDirectory sd, NamespaceInfo nsInfo, StartupOption startOpt) throws IOException { - if (startOpt == StartupOption.ROLLBACK) { + if (startOpt == StartupOption.ROLLBACK && sd.getPreviousDir().exists()) { + // we will already restore everything in the trash by rolling back to + // the previous directory, so we must delete the trash to ensure + // that it's not restored by BPOfferService.signalRollingUpgrade() + if (!FileUtil.fullyDelete(getTrashRootDir(sd))) { + throw new IOException("Unable to delete trash directory prior to " + + "restoration of previous directory: " + getTrashRootDir(sd)); + } doRollback(sd, nsInfo); // rollback if applicable } else { // Restore all the files in the trash. The restored files are retained http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae8178c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 1ec91d0..7edffa4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -244,10 +244,9 @@ public class DataNode extends Configured LogFactory.getLog(DataNode.class.getName() + ".clienttrace"); private static final String USAGE = - "Usage: java DataNode [-regular | -rollback | -rollingupgrade rollback]\n" + + "Usage: java DataNode [-regular | -rollback]\n" + " -regular : Normal DataNode startup (default).\n" + - " -rollback : Rollback a standard upgrade.\n" + - " -rollingupgrade rollback : Rollback a rolling upgrade operation.\n" + + " -rollback : Rollback a standard or rolling upgrade.\n" + " Refer to HDFS documentation for the difference between standard\n" + " and rolling upgrades."; http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae8178c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index e17d403..a0b636f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -1073,6 +1073,7 @@ class NameNodeRpcServer implements NamenodeProtocols { } if (nn.getFSImage().isUpgradeFinalized() && + !namesystem.isRollingUpgrade() && !nn.isStandbyState() && noStaleStorages) { return new FinalizeCommand(poolId); http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae8178c/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml index c369f3b..61d7d06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml @@ -206,7 +206,7 @@ <li>Restore the pre-upgrade release in all machines.</li> <li>Start <em>NNs</em> with the "<a href="#namenode_-rollingUpgrade"><code>-rollingUpgrade rollback</code></a>" option.</li> - <li>Start <em>DNs</em> normally.</li> + <li>Start <em>DNs</em> with the "<code>-rollback</code>" option.</li> </ol></li> </ul>