Repository: hadoop Updated Branches: refs/heads/branch-2.6 0af4f222a -> 530b8a75c
MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state. Contributed by zhihai xu. (cherry picked from commit 1ea90144d58443a7431ff33ba49ee19278ebe42b) Conflicts: hadoop-mapreduce-project/CHANGES.txt hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/530b8a75 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/530b8a75 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/530b8a75 Branch: refs/heads/branch-2.6 Commit: 530b8a75cb3a8a8c122a6026603d4070ea7b7ec4 Parents: 0af4f22 Author: Jason Lowe <jl...@apache.org> Authored: Fri Oct 30 15:15:49 2015 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Fri Oct 30 15:15:49 2015 +0000 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 4 +++ .../mapreduce/v2/hs/HistoryFileManager.java | 11 +++--- .../mapreduce/v2/hs/TestHistoryFileManager.java | 35 ++++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/530b8a75/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 53f938f..ce1e065 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -12,6 +12,10 @@ Release 2.6.3 - UNRELEASED BUG FIXES + MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to + avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state. + (zhihai xu via devaraj) + Release 2.6.2 - 2015-10-21 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/530b8a75/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 2f74d15..d0e171e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -299,8 +299,9 @@ public class HistoryFileManager extends AbstractService { private JobIndexInfo jobIndexInfo; private HistoryInfoState state; - private HistoryFileInfo(Path historyFile, Path confFile, Path summaryFile, - JobIndexInfo jobIndexInfo, boolean isInDone) { + @VisibleForTesting + protected HistoryFileInfo(Path historyFile, Path confFile, + Path summaryFile, JobIndexInfo jobIndexInfo, boolean isInDone) { this.historyFile = historyFile; this.confFile = confFile; this.summaryFile = summaryFile; @@ -333,7 +334,8 @@ public class HistoryFileManager extends AbstractService { + " historyFile = " + historyFile; } - private synchronized void moveToDone() throws IOException { + @VisibleForTesting + synchronized void moveToDone() throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("moveToDone: " + historyFile); } @@ -364,7 +366,8 @@ public class HistoryFileManager extends AbstractService { paths.add(confFile); } - if (summaryFile == null) { + if (summaryFile == null || !intermediateDoneDirFc.util().exists( + summaryFile)) { LOG.info("No summary file for job: " + jobId); } else { String jobSummaryString = getJobSummary(intermediateDoneDirFc, http://git-wip-us.apache.org/repos/asf/hadoop/blob/530b8a75/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java index e2e943a..e5b38ff 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java @@ -31,7 +31,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.TypeConverter; +import org.apache.hadoop.mapreduce.v2.api.records.JobId; +import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo; import org.apache.hadoop.test.CoreTestDriver; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -199,4 +204,34 @@ public class TestHistoryFileManager { testCreateHistoryDirs(dfsCluster.getConfiguration(0), clock); } + @Test + public void testHistoryFileInfoSummaryFileNotExist() throws Exception { + HistoryFileManagerTest hmTest = new HistoryFileManagerTest(); + String job = "job_1410889000000_123456"; + Path summaryFile = new Path(job + ".summary"); + JobIndexInfo jobIndexInfo = new JobIndexInfo(); + jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(job))); + Configuration conf = dfsCluster.getConfiguration(0); + conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, + "/" + UUID.randomUUID()); + conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, + "/" + UUID.randomUUID()); + hmTest.serviceInit(conf); + HistoryFileInfo info = hmTest.getHistoryFileInfo(null, null, + summaryFile, jobIndexInfo, false); + info.moveToDone(); + Assert.assertFalse(info.didMoveFail()); + } + + static class HistoryFileManagerTest extends HistoryFileManager { + public HistoryFileManagerTest() { + super(); + } + public HistoryFileInfo getHistoryFileInfo(Path historyFile, + Path confFile, Path summaryFile, JobIndexInfo jobIndexInfo, + boolean isInDone) { + return new HistoryFileInfo(historyFile, confFile, summaryFile, + jobIndexInfo, isInDone); + } + } }