Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 56882e7b1 -> 03292b234
HDFS-9533. seen_txid in the shared edits directory is modified during
bootstrapping. Contributed by Kihwal Lee.
(cherry picked from commit 5cb1e0118b173a95c1f7bdfae1e58d7833d61c26)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
(cherry picked from commit 3c0ae5914ca673b055a9673b4c17c8aaa570c258)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03292b23
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03292b23
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03292b23
Branch: refs/heads/branch-2.7
Commit: 03292b234e9bb2c7867accbf93a7d11cc894b629
Parents: 56882e7
Author: Kihwal Lee <kih...@apache.org>
Authored: Thu Dec 17 17:28:24 2015 -0600
Committer: Kihwal Lee <kih...@apache.org>
Committed: Thu Dec 17 17:28:24 2015 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/NNStorage.java | 18 +++++++++++++++++-
.../hdfs/server/namenode/ha/BootstrapStandby.java | 6 ++++--
.../hdfs/server/namenode/FSImageTestUtil.java | 7 +++++++
.../server/namenode/ha/TestBootstrapStandby.java | 12 ++++++++++++
5 files changed, 43 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03292b23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b7b3cda..dcaff35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -36,6 +36,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9516. Truncate file fails with data dirs on multiple disks.
(Plamen Jeliazkov via shv)
+ HDFS-9533. seen_txid in the shared edits directory is modified during
+ bootstrapping (kihwal)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03292b23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
index dbb2c50..98699e2 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
@@ -473,8 +473,24 @@ public class NNStorage extends Storage implements
Closeable,
* @param txid the txid that has been reached
*/
public void writeTransactionIdFileToStorage(long txid) {
+ writeTransactionIdFileToStorage(txid, null);
+ }
+
+ /**
+ * Write a small file in all available storage directories that
+ * indicates that the namespace has reached some given transaction ID.
+ *
+ * This is used when the image is loaded to avoid accidental rollbacks
+ * in the case where an edit log is fully deleted but there is no
+ * checkpoint. See TestNameEditsConfigs.testNameEditsConfigsFailure()
+ * @param txid the txid that has been reached
+ * @param type the type of directory
+ */
+ public void writeTransactionIdFileToStorage(long txid,
+ NameNodeDirType type) {
// Write txid marker in all storage directories
- for (StorageDirectory sd : storageDirs) {
+ for (Iterator<StorageDirectory> it = dirIterator(type); it.hasNext();) {
+ StorageDirectory sd = it.next();
try {
writeTransactionIdFile(sd, txid);
} catch(IOException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03292b23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
index 4bd53a5..2d83cba 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.namenode.NNUpgradeUtil;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.TransferFsImage;
@@ -312,13 +313,14 @@ public class BootstrapStandby implements Tool,
Configurable {
return ERR_CODE_LOGS_UNAVAILABLE;
}
- image.getStorage().writeTransactionIdFileToStorage(curTxId);
-
// Download that checkpoint into our storage directories.
MD5Hash hash = TransferFsImage.downloadImageToStorage(
otherHttpAddr, imageTxId, storage, true);
image.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, imageTxId,
hash);
+
+ // Write seen_txid to the formatted image directories.
+ storage.writeTransactionIdFileToStorage(imageTxId,
NameNodeDirType.IMAGE);
} catch (IOException ioe) {
image.close();
throw ioe;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03292b23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
index dd370fa..32a272f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
@@ -568,4 +568,11 @@ public abstract class FSImageTestUtil {
FSImageTestUtil.assertParallelFilesAreIdentical(curDirs,
ignoredFiles);
}
+
+ public static long getStorageTxId(NameNode node, URI storageUri)
+ throws IOException {
+ StorageDirectory sDir = getFSImage(node).getStorage().
+ getStorageDirectory(storageUri);
+ return NNStorage.readTransactionIdFile(sDir);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03292b23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
index 7abc502..85753ad 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
@@ -29,6 +29,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
@@ -126,6 +127,13 @@ public class TestBootstrapStandby {
.getFSImage().getMostRecentCheckpointTxId();
assertEquals(6, expectedCheckpointTxId);
+ // advance the current txid
+ cluster.getFileSystem(0).create(new Path("/test_txid"), (short)1).close();
+
+ // obtain the content of seen_txid
+ URI editsUri = cluster.getSharedEditsDir(0, 1);
+ long seen_txid_shared = FSImageTestUtil.getStorageTxId(nn0, editsUri);
+
int rc = BootstrapStandby.run(
new String[]{"-force"},
cluster.getConfiguration(1));
@@ -136,6 +144,10 @@ public class TestBootstrapStandby {
ImmutableList.of((int)expectedCheckpointTxId));
FSImageTestUtil.assertNNFilesMatch(cluster);
+ // Make sure the seen_txid was not modified by the standby
+ assertEquals(seen_txid_shared,
+ FSImageTestUtil.getStorageTxId(nn0, editsUri));
+
// We should now be able to start the standby successfully.
cluster.restartNameNode(1);
}
