Author: vinayakumarb
Date: Fri Jul 25 07:04:01 2014
New Revision: 1613356
URL: http://svn.apache.org/r1613356
Log:
Merged revision(s) 1613355 from hadoop/common/trunk:
HDFS-5919. FileJournalManager doesn't purge empty and corrupt inprogress edits
files (vinayakumarb)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1613356&r1=1613355&r2=1613356&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri Jul 25 07:04:01 2014
@@ -115,6 +115,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6715. Webhdfs wont fail over when it gets java.io.IOException:
Namenode
is in startup mode. (jing9)
+ HDFS-5919. FileJournalManager doesn't purge empty and corrupt inprogress
edits
+ files (vinayakumarb)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1613356&r1=1613355&r2=1613356&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
Fri Jul 25 07:04:01 2014
@@ -70,6 +70,8 @@ public class FileJournalManager implemen
NameNodeFile.EDITS.getName() + "_(\\d+)-(\\d+)");
private static final Pattern EDITS_INPROGRESS_REGEX = Pattern.compile(
NameNodeFile.EDITS_INPROGRESS.getName() + "_(\\d+)");
+ private static final Pattern EDITS_INPROGRESS_STALE_REGEX = Pattern.compile(
+ NameNodeFile.EDITS_INPROGRESS.getName() + "_(\\d+).*(\\S+)");
private File currentInProgress = null;
@@ -161,8 +163,7 @@ public class FileJournalManager implemen
throws IOException {
LOG.info("Purging logs older than " + minTxIdToKeep);
File[] files = FileUtil.listFiles(sd.getCurrentDir());
- List<EditLogFile> editLogs =
- FileJournalManager.matchEditLogs(files);
+ List<EditLogFile> editLogs = matchEditLogs(files, true);
for (EditLogFile log : editLogs) {
if (log.getFirstTxId() < minTxIdToKeep &&
log.getLastTxId() < minTxIdToKeep) {
@@ -245,6 +246,11 @@ public class FileJournalManager implemen
}
static List<EditLogFile> matchEditLogs(File[] filesInStorage) {
+ return matchEditLogs(filesInStorage, false);
+ }
+
+ private static List<EditLogFile> matchEditLogs(File[] filesInStorage,
+ boolean forPurging) {
List<EditLogFile> ret = Lists.newArrayList();
for (File f : filesInStorage) {
String name = f.getName();
@@ -255,6 +261,7 @@ public class FileJournalManager implemen
long startTxId = Long.parseLong(editsMatch.group(1));
long endTxId = Long.parseLong(editsMatch.group(2));
ret.add(new EditLogFile(f, startTxId, endTxId));
+ continue;
} catch (NumberFormatException nfe) {
LOG.error("Edits file " + f + " has improperly formatted " +
"transaction ID");
@@ -269,12 +276,30 @@ public class FileJournalManager implemen
long startTxId = Long.parseLong(inProgressEditsMatch.group(1));
ret.add(
new EditLogFile(f, startTxId, HdfsConstants.INVALID_TXID, true));
+ continue;
} catch (NumberFormatException nfe) {
LOG.error("In-progress edits file " + f + " has improperly " +
"formatted transaction ID");
// skip
}
}
+ if (forPurging) {
+ // Check for in-progress stale edits
+ Matcher staleInprogressEditsMatch = EDITS_INPROGRESS_STALE_REGEX
+ .matcher(name);
+ if (staleInprogressEditsMatch.matches()) {
+ try {
+ long startTxId = Long.valueOf(staleInprogressEditsMatch.group(1));
+ ret.add(new EditLogFile(f, startTxId, HdfsConstants.INVALID_TXID,
+ true));
+ continue;
+ } catch (NumberFormatException nfe) {
+ LOG.error("In-progress stale edits file " + f + " has improperly "
+ + "formatted transaction ID");
+ // skip
+ }
+ }
+ }
}
return ret;
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java?rev=1613356&r1=1613355&r2=1613356&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
Fri Jul 25 07:04:01 2014
@@ -212,18 +212,25 @@ public class TestNNStorageRetentionManag
tc.addImage("/foo1/current/" + getImageFileName(300), false);
tc.addImage("/foo1/current/" + getImageFileName(400), false);
+ // Segments containing txns upto txId 250 are extra and should be purged.
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(1, 100), true);
- // Without lowering the max segments to retain, we'd retain all segments
- // going back to txid 150 (300 - 150).
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 175), true);
+ tc.addLog("/foo2/current/" + getInProgressEditsFileName(176) + ".empty",
+ true);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(176, 200), true);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 225), true);
+ tc.addLog("/foo2/current/" + getInProgressEditsFileName(226) + ".corrupt",
+ true);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(226, 240), true);
// Only retain 2 extra segments. The 301-350 and 351-400 segments are
// considered required, not extra.
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(241, 275), false);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(276, 300), false);
+ tc.addLog("/foo2/current/" + getInProgressEditsFileName(301) + ".empty",
+ false);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 350), false);
+ tc.addLog("/foo2/current/" + getInProgressEditsFileName(351) + ".corrupt",
+ false);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(351, 400), false);
tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false);
runTest(tc);
