MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state. Contributed by zhihai xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ea90144 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ea90144 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ea90144 Branch: refs/heads/HDFS-7240 Commit: 1ea90144d58443a7431ff33ba49ee19278ebe42b Parents: 3bef7c8 Author: Devaraj K <deva...@apache.org> Authored: Fri May 15 12:31:32 2015 +0530 Committer: Devaraj K <deva...@apache.org> Committed: Fri May 15 12:31:32 2015 +0530 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 4 +++ .../mapreduce/v2/hs/HistoryFileManager.java | 11 +++--- .../mapreduce/v2/hs/TestHistoryFileManager.java | 35 ++++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea90144/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index bc1f427..7e34297 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -425,6 +425,10 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-5708. Duplicate String.format in YarnOutputFiles.getSpillFileForWrite. (Konstantin Weitz via devaraj) + MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to + avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state. + (zhihai xu via devaraj) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea90144/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 69f814d..f0786da 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -303,8 +303,9 @@ public class HistoryFileManager extends AbstractService { private JobIndexInfo jobIndexInfo; private HistoryInfoState state; - private HistoryFileInfo(Path historyFile, Path confFile, Path summaryFile, - JobIndexInfo jobIndexInfo, boolean isInDone) { + @VisibleForTesting + protected HistoryFileInfo(Path historyFile, Path confFile, + Path summaryFile, JobIndexInfo jobIndexInfo, boolean isInDone) { this.historyFile = historyFile; this.confFile = confFile; this.summaryFile = summaryFile; @@ -337,7 +338,8 @@ public class HistoryFileManager extends AbstractService { + " historyFile = " + historyFile; } - private synchronized void moveToDone() throws IOException { + @VisibleForTesting + synchronized void moveToDone() throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("moveToDone: " + historyFile); } @@ -368,7 +370,8 @@ public class HistoryFileManager extends AbstractService { paths.add(confFile); } - if (summaryFile == null) { + if (summaryFile == null || !intermediateDoneDirFc.util().exists( + summaryFile)) { LOG.info("No summary file for job: " + jobId); } else { String jobSummaryString = getJobSummary(intermediateDoneDirFc, http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea90144/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java index 1c5cc5c..7694de9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java @@ -35,7 +35,12 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.TypeConverter; +import org.apache.hadoop.mapreduce.v2.api.records.JobId; +import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo; import org.apache.hadoop.test.CoreTestDriver; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -221,4 +226,34 @@ public class TestHistoryFileManager { } + @Test + public void testHistoryFileInfoSummaryFileNotExist() throws Exception { + HistoryFileManagerTest hmTest = new HistoryFileManagerTest(); + String job = "job_1410889000000_123456"; + Path summaryFile = new Path(job + ".summary"); + JobIndexInfo jobIndexInfo = new JobIndexInfo(); + jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(job))); + Configuration conf = dfsCluster.getConfiguration(0); + conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, + "/" + UUID.randomUUID()); + conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, + "/" + UUID.randomUUID()); + hmTest.serviceInit(conf); + HistoryFileInfo info = hmTest.getHistoryFileInfo(null, null, + summaryFile, jobIndexInfo, false); + info.moveToDone(); + Assert.assertFalse(info.didMoveFail()); + } + + static class HistoryFileManagerTest extends HistoryFileManager { + public HistoryFileManagerTest() { + super(); + } + public HistoryFileInfo getHistoryFileInfo(Path historyFile, + Path confFile, Path summaryFile, JobIndexInfo jobIndexInfo, + boolean isInDone) { + return new HistoryFileInfo(historyFile, confFile, summaryFile, + jobIndexInfo, isInDone); + } + } }