HDFS-12836. startTxId could be greater than endTxId when tailing in-progress edit log. Contributed by Chao Sun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0faf5062 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0faf5062 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0faf5062 Branch: refs/heads/HDFS-7240 Commit: 0faf50624580b86b64a828cdbbb630ae8994e2cd Parents: 53bbef3 Author: Wei-Chiu Chuang <[email protected]> Authored: Fri Dec 1 12:01:21 2017 -0800 Committer: Wei-Chiu Chuang <[email protected]> Committed: Fri Dec 1 12:01:21 2017 -0800 ---------------------------------------------------------------------- .../qjournal/client/QuorumJournalManager.java | 6 ++++++ .../namenode/ha/TestStandbyInProgressTail.java | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0faf5062/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java index d30625b..7dff9b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java @@ -498,6 +498,12 @@ public class QuorumJournalManager implements JournalManager { // than committedTxnId. This ensures the consistency. if (onlyDurableTxns && inProgressOk) { endTxId = Math.min(endTxId, committedTxnId); + if (endTxId < remoteLog.getStartTxId()) { + LOG.warn("Found endTxId (" + endTxId + ") that is less than " + + "the startTxId (" + remoteLog.getStartTxId() + + ") - setting it to startTxId."); + endTxId = remoteLog.getStartTxId(); + } } EditLogInputStream elis = EditLogFileInputStream.fromUrl( http://git-wip-us.apache.org/repos/asf/hadoop/blob/0faf5062/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyInProgressTail.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyInProgressTail.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyInProgressTail.java index 9201cda..b1cd037 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyInProgressTail.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyInProgressTail.java @@ -309,6 +309,25 @@ public class TestStandbyInProgressTail { assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test3", true)); } + @Test + public void testNonUniformConfig() throws Exception { + // Test case where some NNs (in this case the active NN) in the cluster + // do not have in-progress tailing enabled. + Configuration newConf = cluster.getNameNode(0).getConf(); + newConf.setBoolean( + DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, + false); + cluster.restartNameNode(0); + cluster.transitionToActive(0); + + cluster.getNameNode(0).getRpcServer().mkdirs("/test", + FsPermission.createImmutable((short) 0755), true); + cluster.getNameNode(0).getRpcServer().rollEdits(); + + cluster.getNameNode(1).getNamesystem().getEditLogTailer().doTailEdits(); + assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test", true)); + } + /** * Check that no edits files are present in the given storage dirs. */ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
