Repository: hadoop Updated Branches: refs/heads/trunk a20e7105e -> d04f85f38
MAPREDUCE-6892. Issues with the count of failed/killed tasks in the jhist file. (Peter Bacsko via Haibo Chen) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d04f85f3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d04f85f3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d04f85f3 Branch: refs/heads/trunk Commit: d04f85f387e4a78816bc9966ee2b4a647ee05faf Parents: a20e710 Author: Haibo Chen <haiboc...@apache.org> Authored: Wed Aug 30 10:07:48 2017 -0700 Committer: Haibo Chen <haiboc...@apache.org> Committed: Wed Aug 30 10:07:48 2017 -0700 ---------------------------------------------------------------------- .../jobhistory/JobHistoryEventHandler.java | 78 +++++++++--- .../hadoop/mapreduce/jobhistory/JobSummary.java | 49 ++++++-- .../apache/hadoop/mapreduce/v2/app/job/Job.java | 4 + .../mapreduce/v2/app/job/impl/JobImpl.java | 43 ++++++- .../jobhistory/TestJobHistoryEventHandler.java | 41 ++++--- .../mapreduce/jobhistory/TestJobSummary.java | 6 +- .../hadoop/mapreduce/v2/app/MockJobs.java | 19 +++ .../mapreduce/v2/app/TestRuntimeEstimators.java | 20 +++ .../src/main/avro/Events.avpr | 10 +- .../mapreduce/jobhistory/HistoryViewer.java | 18 +-- .../HumanReadableHistoryViewerPrinter.java | 4 +- .../jobhistory/JSONHistoryViewerPrinter.java | 4 +- .../mapreduce/jobhistory/JobFinishedEvent.java | 55 ++++++--- .../mapreduce/jobhistory/JobHistoryParser.java | 34 ++++-- .../JobUnsuccessfulCompletionEvent.java | 78 +++++++++--- .../jobhistory/TestHistoryViewerPrinter.java | 4 +- .../hadoop/mapreduce/v2/hs/CompletedJob.java | 55 ++++++++- .../hadoop/mapreduce/v2/hs/PartialJob.java | 19 +++ .../hadoop/mapreduce/v2/hs/UnparsedJob.java | 20 +++ .../mapreduce/v2/hs/TestJobHistoryParsing.java | 122 +++++++++++++++++-- .../v2/hs/webapp/TestHsWebServicesAcls.java | 20 +++ .../rumen/Job20LineHistoryEventEmitter.java | 6 +- 22 files changed, 573 insertions(+), 136 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index cfa91f5..8fa417b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -431,10 +431,18 @@ public class JobHistoryEventHandler extends AbstractService + " to have not been closed. Will close"); //Create a JobFinishEvent so that it is written to the job history final Job job = context.getJob(toClose); + int successfulMaps = job.getCompletedMaps() - job.getFailedMaps() + - job.getKilledMaps(); + int successfulReduces = job.getCompletedReduces() + - job.getFailedReduces() - job.getKilledReduces(); + JobUnsuccessfulCompletionEvent jucEvent = new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(toClose), - System.currentTimeMillis(), job.getCompletedMaps(), - job.getCompletedReduces(), + System.currentTimeMillis(), + successfulMaps, + successfulReduces, + job.getFailedMaps(), job.getFailedReduces(), + job.getKilledMaps(), job.getKilledReduces(), createJobStateForJobUnsuccessfulCompletionEvent( mi.getForcedJobStateOnShutDown()), job.getDiagnostics()); @@ -655,9 +663,9 @@ public class JobHistoryEventHandler extends AbstractService JobFinishedEvent jFinishedEvent = (JobFinishedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setFinishTime(jFinishedEvent.getFinishTime()); - mi.getJobIndexInfo().setNumMaps(jFinishedEvent.getFinishedMaps()); + mi.getJobIndexInfo().setNumMaps(jFinishedEvent.getSucceededMaps()); mi.getJobIndexInfo().setNumReduces( - jFinishedEvent.getFinishedReduces()); + jFinishedEvent.getSucceededReduces()); mi.getJobIndexInfo().setJobStatus(JobState.SUCCEEDED.toString()); closeEventWriter(event.getJobID()); processDoneFiles(event.getJobID()); @@ -672,8 +680,8 @@ public class JobHistoryEventHandler extends AbstractService JobUnsuccessfulCompletionEvent jucEvent = (JobUnsuccessfulCompletionEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setFinishTime(jucEvent.getFinishTime()); - mi.getJobIndexInfo().setNumMaps(jucEvent.getFinishedMaps()); - mi.getJobIndexInfo().setNumReduces(jucEvent.getFinishedReduces()); + mi.getJobIndexInfo().setNumMaps(jucEvent.getSucceededMaps()); + mi.getJobIndexInfo().setNumReduces(jucEvent.getSucceededReduces()); mi.getJobIndexInfo().setJobStatus(jucEvent.getStatus()); closeEventWriter(event.getJobID()); if(context.isLastAMRetry()) @@ -690,8 +698,8 @@ public class JobHistoryEventHandler extends AbstractService (JobUnsuccessfulCompletionEvent) event .getHistoryEvent(); mi.getJobIndexInfo().setFinishTime(jucEvent.getFinishTime()); - mi.getJobIndexInfo().setNumMaps(jucEvent.getFinishedMaps()); - mi.getJobIndexInfo().setNumReduces(jucEvent.getFinishedReduces()); + mi.getJobIndexInfo().setNumMaps(jucEvent.getSucceededMaps()); + mi.getJobIndexInfo().setNumReduces(jucEvent.getSucceededReduces()); mi.getJobIndexInfo().setJobStatus(jucEvent.getStatus()); closeEventWriter(event.getJobID()); processDoneFiles(event.getJobID()); @@ -739,10 +747,12 @@ public class JobHistoryEventHandler extends AbstractService case JOB_FINISHED: JobFinishedEvent jfe = (JobFinishedEvent) event; summary.setJobFinishTime(jfe.getFinishTime()); - summary.setNumFinishedMaps(jfe.getFinishedMaps()); + summary.setNumSucceededMaps(jfe.getSucceededMaps()); summary.setNumFailedMaps(jfe.getFailedMaps()); - summary.setNumFinishedReduces(jfe.getFinishedReduces()); + summary.setNumSucceededReduces(jfe.getSucceededReduces()); summary.setNumFailedReduces(jfe.getFailedReduces()); + summary.setNumKilledMaps(jfe.getKilledMaps()); + summary.setNumKilledReduces(jfe.getKilledReduces()); if (summary.getJobStatus() == null) summary .setJobStatus(org.apache.hadoop.mapreduce.JobStatus.State.SUCCEEDED @@ -753,11 +763,21 @@ public class JobHistoryEventHandler extends AbstractService break; case JOB_FAILED: case JOB_KILLED: + Job job = context.getJob(jobId); JobUnsuccessfulCompletionEvent juce = (JobUnsuccessfulCompletionEvent) event; + int successfulMaps = job.getCompletedMaps() - job.getFailedMaps() + - job.getKilledMaps(); + int successfulReduces = job.getCompletedReduces() + - job.getFailedReduces() - job.getKilledReduces(); + summary.setJobStatus(juce.getStatus()); - summary.setNumFinishedMaps(context.getJob(jobId).getTotalMaps()); - summary.setNumFinishedReduces(context.getJob(jobId).getTotalReduces()); + summary.setNumSucceededMaps(successfulMaps); + summary.setNumSucceededReduces(successfulReduces); + summary.setNumFailedMaps(job.getFailedMaps()); + summary.setNumFailedReduces(job.getFailedReduces()); summary.setJobFinishTime(juce.getFinishTime()); + summary.setNumKilledMaps(juce.getKilledMaps()); + summary.setNumKilledReduces(juce.getKilledReduces()); setSummarySlotSeconds(summary, context.getJob(jobId).getAllCounters()); break; default: @@ -840,12 +860,22 @@ public class JobHistoryEventHandler extends AbstractService JobUnsuccessfulCompletionEvent juce = (JobUnsuccessfulCompletionEvent) event; tEvent.addEventInfo("FINISH_TIME", juce.getFinishTime()); - tEvent.addEventInfo("NUM_MAPS", juce.getFinishedMaps()); - tEvent.addEventInfo("NUM_REDUCES", juce.getFinishedReduces()); + tEvent.addEventInfo("NUM_MAPS", + juce.getSucceededMaps() + + juce.getFailedMaps() + + juce.getKilledMaps()); + tEvent.addEventInfo("NUM_REDUCES", + juce.getSucceededReduces() + + juce.getFailedReduces() + + juce.getKilledReduces()); tEvent.addEventInfo("JOB_STATUS", juce.getStatus()); tEvent.addEventInfo("DIAGNOSTICS", juce.getDiagnostics()); - tEvent.addEventInfo("FINISHED_MAPS", juce.getFinishedMaps()); - tEvent.addEventInfo("FINISHED_REDUCES", juce.getFinishedReduces()); + tEvent.addEventInfo("SUCCESSFUL_MAPS", juce.getSucceededMaps()); + tEvent.addEventInfo("SUCCESSFUL_REDUCES", juce.getSucceededReduces()); + tEvent.addEventInfo("FAILED_MAPS", juce.getFailedMaps()); + tEvent.addEventInfo("FAILED_REDUCES", juce.getFailedReduces()); + tEvent.addEventInfo("KILLED_MAPS", juce.getKilledMaps()); + tEvent.addEventInfo("KILLED_REDUCES", juce.getKilledReduces()); tEntity.addEvent(tEvent); tEntity.setEntityId(jobId.toString()); tEntity.setEntityType(MAPREDUCE_JOB_ENTITY_TYPE); @@ -853,12 +883,20 @@ public class JobHistoryEventHandler extends AbstractService case JOB_FINISHED: JobFinishedEvent jfe = (JobFinishedEvent) event; tEvent.addEventInfo("FINISH_TIME", jfe.getFinishTime()); - tEvent.addEventInfo("NUM_MAPS", jfe.getFinishedMaps()); - tEvent.addEventInfo("NUM_REDUCES", jfe.getFinishedReduces()); + tEvent.addEventInfo("NUM_MAPS", + jfe.getSucceededMaps() + + jfe.getFailedMaps() + + jfe.getKilledMaps()); + tEvent.addEventInfo("NUM_REDUCES", + jfe.getSucceededReduces() + + jfe.getFailedReduces() + + jfe.getKilledReduces()); tEvent.addEventInfo("FAILED_MAPS", jfe.getFailedMaps()); tEvent.addEventInfo("FAILED_REDUCES", jfe.getFailedReduces()); - tEvent.addEventInfo("FINISHED_MAPS", jfe.getFinishedMaps()); - tEvent.addEventInfo("FINISHED_REDUCES", jfe.getFinishedReduces()); + tEvent.addEventInfo("SUCCESSFUL_MAPS", jfe.getSucceededMaps()); + tEvent.addEventInfo("SUCCESSFUL_REDUCES", jfe.getSucceededReduces()); + tEvent.addEventInfo("KILLED_MAPS", jfe.getKilledMaps()); + tEvent.addEventInfo("KILLED_REDUCES", jfe.getKilledReduces()); tEvent.addEventInfo("MAP_COUNTERS_GROUPS", JobHistoryEventUtils.countersToJSON(jfe.getMapCounters())); tEvent.addEventInfo("REDUCE_COUNTERS_GROUPS", http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSummary.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSummary.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSummary.java index abe9518..22ae079 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSummary.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSummary.java @@ -30,10 +30,12 @@ public class JobSummary { private long firstReduceTaskLaunchTime; // ReduceAttemptStarted | // TaskAttemptStartEvent private long jobFinishTime; - private int numFinishedMaps; + private int numSucceededMaps; private int numFailedMaps; - private int numFinishedReduces; + private int numSucceededReduces; private int numFailedReduces; + private int numKilledMaps; + private int numKilledReduces; private int resourcesPerMap; // resources used per map/min resource private int resourcesPerReduce; // resources used per reduce/min resource // resource models @@ -98,12 +100,12 @@ public class JobSummary { this.jobFinishTime = jobFinishTime; } - public int getNumFinishedMaps() { - return numFinishedMaps; + public int getNumSucceededMaps() { + return numSucceededMaps; } - public void setNumFinishedMaps(int numFinishedMaps) { - this.numFinishedMaps = numFinishedMaps; + public void setNumSucceededMaps(int numSucceededMaps) { + this.numSucceededMaps = numSucceededMaps; } public int getNumFailedMaps() { @@ -114,6 +116,22 @@ public class JobSummary { this.numFailedMaps = numFailedMaps; } + public int getKilledMaps() { + return numKilledMaps; + } + + public void setNumKilledMaps(int numKilledMaps) { + this.numKilledMaps = numKilledMaps; + } + + public int getKilledReduces() { + return numKilledReduces; + } + + public void setNumKilledReduces(int numKilledReduces) { + this.numKilledReduces = numKilledReduces; + } + public int getResourcesPerMap() { return resourcesPerMap; } @@ -122,12 +140,12 @@ public class JobSummary { this.resourcesPerMap = resourcesPerMap; } - public int getNumFinishedReduces() { - return numFinishedReduces; + public int getNumSucceededReduces() { + return numSucceededReduces; } - public void setNumFinishedReduces(int numFinishedReduces) { - this.numFinishedReduces = numFinishedReduces; + public void setNumSucceededReduces(int numSucceededReduces) { + this.numSucceededReduces = numSucceededReduces; } public int getNumFailedReduces() { @@ -204,8 +222,15 @@ public class JobSummary { .add("finishTime", jobFinishTime) .add("resourcesPerMap", resourcesPerMap) .add("resourcesPerReduce", resourcesPerReduce) - .add("numMaps", numFinishedMaps + numFailedMaps) - .add("numReduces", numFinishedReduces + numFailedReduces) + .add("numMaps", numSucceededMaps + numFailedMaps + numKilledMaps) + .add("numReduces", numSucceededReduces + numFailedReduces + + numKilledReduces) + .add("succededMaps", numSucceededMaps) + .add("succeededReduces", numSucceededReduces) + .add("failedMaps", numFailedMaps) + .add("failedReduces", numFailedReduces) + .add("killedMaps", numKilledMaps) + .add("killedReduces", numKilledReduces) .add("user", user) .add("queue", queue) .add("status", jobStatus) http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java index 7738810..437707b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java @@ -65,6 +65,10 @@ public interface Job { int getTotalReduces(); int getCompletedMaps(); int getCompletedReduces(); + int getFailedMaps(); + int getFailedReduces(); + int getKilledMaps(); + int getKilledReduces(); float getProgress(); boolean isUber(); String getUserName(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 6880b6c..abc3e61 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -1684,6 +1684,10 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, finishTime, succeededMapTaskCount, succeededReduceTaskCount, + failedMapTaskCount, + failedReduceTaskCount, + killedMapTaskCount, + killedReduceTaskCount, finalState.toString(), diagnostics); eventHandler.handle(new JobHistoryEvent(jobId, @@ -1748,6 +1752,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, job.oldJobId, job.finishTime, job.succeededMapTaskCount, job.succeededReduceTaskCount, job.failedMapTaskCount, job.failedReduceTaskCount, + job.killedMapTaskCount, job.killedReduceTaskCount, job.finalMapCounters, job.finalReduceCounters, job.fullCounters); @@ -1797,7 +1802,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, job.setFinishTime(); JobUnsuccessfulCompletionEvent failedEvent = new JobUnsuccessfulCompletionEvent(job.oldJobId, - job.finishTime, 0, 0, + job.finishTime, 0, 0, 0, 0, 0, 0, JobStateInternal.KILLED.toString(), job.diagnostics); job.eventHandler.handle(new JobHistoryEvent(job.jobId, failedEvent)); job.finished(JobStateInternal.KILLED); @@ -1954,8 +1959,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, @Override public JobStateInternal transition(JobImpl job, JobEvent event) { job.completedTaskCount++; - LOG.info("Num completed Tasks: " + job.completedTaskCount); JobTaskEvent taskEvent = (JobTaskEvent) event; + LOG.info("Num completed Tasks: " + job.completedTaskCount); Task task = job.tasks.get(taskEvent.getTaskID()); if (taskEvent.getState() == TaskState.SUCCEEDED) { taskSucceeded(job, task); @@ -1991,11 +1996,15 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, job.allowedMapFailuresPercent*job.numMapTasks || job.failedReduceTaskCount*100 > job.allowedReduceFailuresPercent*job.numReduceTasks) { + job.setFinishTime(); String diagnosticMsg = "Job failed as tasks failed. " + "failedMaps:" + job.failedMapTaskCount + - " failedReduces:" + job.failedReduceTaskCount; + " failedReduces:" + job.failedReduceTaskCount + + " killedMaps:" + job.killedMapTaskCount + + " killedReduces: " + job.killedReduceTaskCount; + LOG.info(diagnosticMsg); job.addDiagnostic(diagnosticMsg); @@ -2226,7 +2235,13 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, job.setFinishTime(); JobUnsuccessfulCompletionEvent failedEvent = new JobUnsuccessfulCompletionEvent(job.oldJobId, - job.finishTime, 0, 0, + job.finishTime, + job.succeededMapTaskCount, + job.succeededReduceTaskCount, + job.failedMapTaskCount, + job.failedReduceTaskCount, + job.killedMapTaskCount, + job.killedReduceTaskCount, jobHistoryString, job.diagnostics); job.eventHandler.handle(new JobHistoryEvent(job.jobId, failedEvent)); job.finished(terminationState); @@ -2266,4 +2281,24 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, public void setJobPriority(Priority priority) { this.jobPriority = priority; } + + @Override + public int getFailedMaps() { + return failedMapTaskCount; + } + + @Override + public int getFailedReduces() { + return failedReduceTaskCount; + } + + @Override + public int getKilledMaps() { + return killedMapTaskCount; + } + + @Override + public int getKilledReduces() { + return killedReduceTaskCount; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index e35a84d..47caf44 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -275,7 +275,8 @@ public class TestJobHistoryEventHandler { t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null, 0))); } queueEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( - TypeConverter.fromYarn(t.jobId), 0, 10, 10, 0, 0, null, null, new Counters()))); + TypeConverter.fromYarn(t.jobId), 0, 10, 10, 0, 0, 0, 0, null, null, + new Counters()))); handleNextNEvents(jheh, 29); verify(mockWriter, times(0)).flush(); @@ -308,22 +309,22 @@ public class TestJobHistoryEventHandler { handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(t.jobId), 0, - 0, 0, JobStateInternal.ERROR.toString()))); + 0, 0, 0, 0, 0, 0, JobStateInternal.ERROR.toString()))); verify(jheh, times(1)).processDoneFiles(any(JobId.class)); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( - TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), - new Counters(), new Counters()))); + TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), + new Counters(), new Counters()))); verify(jheh, times(2)).processDoneFiles(any(JobId.class)); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(t.jobId), 0, - 0, 0, JobStateInternal.FAILED.toString()))); + 0, 0, 0, 0, 0, 0, JobStateInternal.FAILED.toString()))); verify(jheh, times(3)).processDoneFiles(any(JobId.class)); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(t.jobId), 0, - 0, 0, JobStateInternal.KILLED.toString()))); + 0, 0, 0, 0, 0, 0, JobStateInternal.KILLED.toString()))); verify(jheh, times(4)).processDoneFiles(any(JobId.class)); mockWriter = jheh.getEventWriter(); @@ -354,22 +355,22 @@ public class TestJobHistoryEventHandler { // skip processing done files handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(t.jobId), 0, - 0, 0, JobStateInternal.ERROR.toString()))); + 0, 0, 0, 0, 0, 0, JobStateInternal.ERROR.toString()))); verify(jheh, times(0)).processDoneFiles(t.jobId); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( - TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), + TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters()))); verify(jheh, times(1)).processDoneFiles(t.jobId); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(t.jobId), 0, - 0, 0, JobStateInternal.FAILED.toString()))); + 0, 0, 0, 0, 0, 0, JobStateInternal.FAILED.toString()))); verify(jheh, times(2)).processDoneFiles(t.jobId); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(t.jobId), 0, - 0, 0, JobStateInternal.KILLED.toString()))); + 0, 0, 0, 0, 0, 0, JobStateInternal.KILLED.toString()))); verify(jheh, times(3)).processDoneFiles(t.jobId); mockWriter = jheh.getEventWriter(); @@ -405,7 +406,8 @@ public class TestJobHistoryEventHandler { "nmhost", 3000, 4000, -1))); handleEvent(jheh, new JobHistoryEvent(params.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn( - params.jobId), 0, 0, 0, JobStateInternal.FAILED.toString()))); + params.jobId), 0, 0, 0, 0, 0, 0, 0, + JobStateInternal.FAILED.toString()))); // verify the value of the sensitive property in job.xml is restored. Assert.assertEquals(sensitivePropertyName + " is modified.", @@ -476,7 +478,7 @@ public class TestJobHistoryEventHandler { t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000, -1))); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( - TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), + TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters()))); // If we got here then event handler worked but we don't know with which @@ -546,7 +548,7 @@ public class TestJobHistoryEventHandler { handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(t.jobId), 0, - 0, 0, JobStateInternal.FAILED.toString()))); + 0, 0, 0, 0, 0, 0, JobStateInternal.FAILED.toString()))); Assert.assertEquals(mi.getJobIndexInfo().getSubmitTime(), 100); Assert.assertEquals(mi.getJobIndexInfo().getJobStartTime(), 200); @@ -642,7 +644,7 @@ public class TestJobHistoryEventHandler { handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent(TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, - 0, new Counters(), new Counters(), new Counters()), currentTime)); + 0, 0, 0, new Counters(), new Counters(), new Counters()), currentTime)); entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null); Assert.assertEquals(1, entities.getEntities().size()); @@ -668,7 +670,8 @@ public class TestJobHistoryEventHandler { handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(t.jobId), - 0, 0, 0, JobStateInternal.KILLED.toString()), currentTime + 20)); + 0, 0, 0, 0, 0, 0, 0, JobStateInternal.KILLED.toString()), + currentTime + 20)); entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null); Assert.assertEquals(1, entities.getEntities().size()); @@ -944,7 +947,7 @@ public class TestJobHistoryEventHandler { // Job finishes and successfully writes history handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( - TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), + TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters()))); verify(jheh, times(1)).processDoneFiles(any(JobId.class)); @@ -978,7 +981,7 @@ public class TestJobHistoryEventHandler { // Job finishes, but doesn't successfully write history handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( - TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), + TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters()))); verify(jheh, times(1)).processDoneFiles(any(JobId.class)); verify(t.mockAppContext, times(0)).setHistoryUrl(any(String.class)); @@ -1009,8 +1012,8 @@ public class TestJobHistoryEventHandler { // Job finishes, but doesn't successfully write history try { handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( - TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), - new Counters(), new Counters()))); + TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, + new Counters(), new Counters(), new Counters()))); throw new RuntimeException( "processDoneFiles didn't throw, but should have"); } catch (YarnRuntimeException yre) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java index 1bea5c8..b6d8bbf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java @@ -43,10 +43,12 @@ public class TestJobSummary { summary.setFirstMapTaskLaunchTime(4L); summary.setFirstReduceTaskLaunchTime(5L); summary.setJobFinishTime(6L); - summary.setNumFinishedMaps(1); + summary.setNumSucceededMaps(1); summary.setNumFailedMaps(0); - summary.setNumFinishedReduces(1); + summary.setNumSucceededReduces(1); summary.setNumFailedReduces(0); + summary.setNumKilledMaps(0); + summary.setNumKilledReduces(0); summary.setUser("testUser"); summary.setQueue("testQueue"); summary.setJobStatus("testJobStatus"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index ccacf1c..bfb8d79 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -640,6 +640,25 @@ public class MockJobs extends MockApps { public void setJobPriority(Priority priority) { // do nothing } + + public int getFailedMaps() { + return 0; + } + + @Override + public int getFailedReduces() { + return 0; + } + + @Override + public int getKilledMaps() { + return 0; + } + + @Override + public int getKilledReduces() { + return 0; + } }; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index 301d498..e1fa198 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -533,6 +533,26 @@ public class TestRuntimeEstimators { public void setJobPriority(Priority priority) { // do nothing } + + @Override + public int getFailedMaps() { + return 0; + } + + @Override + public int getFailedReduces() { + return 0; + } + + @Override + public int getKilledMaps() { + return 0; + } + + @Override + public int getKilledReduces() { + return 0; + } } /* http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr index c7b3eb8..b5a4d87 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr @@ -54,7 +54,9 @@ {"name": "failedReduces", "type": "int"}, {"name": "totalCounters", "type": "JhCounters"}, {"name": "mapCounters", "type": "JhCounters"}, - {"name": "reduceCounters", "type": "JhCounters"} + {"name": "reduceCounters", "type": "JhCounters"}, + {"name": "killedMaps", "type": "int", "default": -1}, + {"name": "killedReduces", "type": "int", "default": -1} ] }, @@ -136,7 +138,11 @@ {"name": "finishedMaps", "type": "int"}, {"name": "finishedReduces", "type": "int"}, {"name": "jobStatus", "type": "string"}, - {"name": "diagnostics", "type": ["null","string"], "default": null} + {"name": "diagnostics", "type": ["null","string"], "default": null}, + {"name": "failedMaps", "type": "int", "default": -1}, + {"name": "failedReduces", "type": "int", "default": -1}, + {"name": "killedMaps", "type": "int", "default": -1}, + {"name": "killedReduces", "type": "int", "default": -1} ] }, http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java index 5f10fdf..c25c73e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java @@ -327,12 +327,12 @@ public class HistoryViewer { /** Generate analysis information for the parsed job */ public AnalyzedJob (JobInfo job) { Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); - int finishedMaps = (int) job.getFinishedMaps(); - int finishedReduces = (int) job.getFinishedReduces(); + int succeededMaps = (int) job.getSucceededMaps(); + int succeededReduces = (int) job.getSucceededReduces(); mapTasks = - new JobHistoryParser.TaskAttemptInfo[finishedMaps]; + new JobHistoryParser.TaskAttemptInfo[succeededMaps]; reduceTasks = - new JobHistoryParser.TaskAttemptInfo[finishedReduces]; + new JobHistoryParser.TaskAttemptInfo[succeededReduces]; int mapIndex = 0 , reduceIndex=0; avgMapTime = 0; avgReduceTime = 0; @@ -360,12 +360,12 @@ public class HistoryViewer { } } } - if (finishedMaps > 0) { - avgMapTime /= finishedMaps; + if (succeededMaps > 0) { + avgMapTime /= succeededMaps; } - if (finishedReduces > 0) { - avgReduceTime /= finishedReduces; - avgShuffleTime /= finishedReduces; + if (succeededReduces > 0) { + avgReduceTime /= succeededReduces; + avgShuffleTime /= succeededReduces; } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java index d3da9f4..685fa05 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java @@ -236,7 +236,7 @@ class HumanReadableHistoryViewerPrinter implements HistoryViewerPrinter { taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff( dateFormat, ts.setupFinished, ts.setupStarted)); taskSummary.append("\nMap\t").append(ts.totalMaps); - taskSummary.append("\t").append(job.getFinishedMaps()); + taskSummary.append("\t").append(job.getSucceededMaps()); taskSummary.append("\t\t").append(ts.numFailedMaps); taskSummary.append("\t").append(ts.numKilledMaps); taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff( @@ -244,7 +244,7 @@ class HumanReadableHistoryViewerPrinter implements HistoryViewerPrinter { taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff( dateFormat, ts.mapFinished, ts.mapStarted)); taskSummary.append("\nReduce\t").append(ts.totalReduces); - taskSummary.append("\t").append(job.getFinishedReduces()); + taskSummary.append("\t").append(job.getSucceededReduces()); taskSummary.append("\t\t").append(ts.numFailedReduces); taskSummary.append("\t").append(ts.numKilledReduces); taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff( http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java index 456dcf7..cfb6641 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java @@ -145,7 +145,7 @@ class JSONHistoryViewerPrinter implements HistoryViewerPrinter { jSums.put("setup", jSumSetup); JSONObject jSumMap = new JSONObject(); jSumMap.put("total", ts.totalMaps); - jSumMap.put("successful", job.getFinishedMaps()); + jSumMap.put("successful", job.getSucceededMaps()); jSumMap.put("failed", ts.numFailedMaps); jSumMap.put("killed", ts.numKilledMaps); jSumMap.put("startTime", ts.mapStarted); @@ -153,7 +153,7 @@ class JSONHistoryViewerPrinter implements HistoryViewerPrinter { jSums.put("map", jSumMap); JSONObject jSumReduce = new JSONObject(); jSumReduce.put("total", ts.totalReduces); - jSumReduce.put("successful", job.getFinishedReduces()); + jSumReduce.put("successful", job.getSucceededReduces()); jSumReduce.put("failed", ts.numFailedReduces); jSumReduce.put("killed", ts.numKilledReduces); jSumReduce.put("startTime", ts.reduceStarted); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java index ea21f60..34e4b2c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java @@ -36,16 +36,18 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class JobFinishedEvent implements HistoryEvent { +public class JobFinishedEvent implements HistoryEvent { private JobFinished datum = null; private JobID jobId; private long finishTime; - private int finishedMaps; - private int finishedReduces; + private int succeededMaps; + private int succeededReduces; private int failedMaps; private int failedReduces; + private int killedMaps; + private int killedReduces; private Counters mapCounters; private Counters reduceCounters; private Counters totalCounters; @@ -54,8 +56,8 @@ public class JobFinishedEvent implements HistoryEvent { * Create an event to record successful job completion * @param id Job ID * @param finishTime Finish time of the job - * @param finishedMaps The number of finished maps - * @param finishedReduces The number of finished reduces + * @param succeededMaps The number of succeeded maps + * @param succeededReduces The number of succeeded reduces * @param failedMaps The number of failed maps * @param failedReduces The number of failed reduces * @param mapCounters Map Counters for the job @@ -63,16 +65,19 @@ public class JobFinishedEvent implements HistoryEvent { * @param totalCounters Total Counters for the job */ public JobFinishedEvent(JobID id, long finishTime, - int finishedMaps, int finishedReduces, + int succeededMaps, int succeededReduces, int failedMaps, int failedReduces, + int killedMaps, int killedReduces, Counters mapCounters, Counters reduceCounters, Counters totalCounters) { this.jobId = id; this.finishTime = finishTime; - this.finishedMaps = finishedMaps; - this.finishedReduces = finishedReduces; + this.succeededMaps = succeededMaps; + this.succeededReduces = succeededReduces; this.failedMaps = failedMaps; this.failedReduces = failedReduces; + this.killedMaps = killedMaps; + this.killedReduces = killedReduces; this.mapCounters = mapCounters; this.reduceCounters = reduceCounters; this.totalCounters = totalCounters; @@ -85,10 +90,14 @@ public class JobFinishedEvent implements HistoryEvent { datum = new JobFinished(); datum.setJobid(new Utf8(jobId.toString())); datum.setFinishTime(finishTime); - datum.setFinishedMaps(finishedMaps); - datum.setFinishedReduces(finishedReduces); + // using finishedMaps & finishedReduces in the Avro schema for backward + // compatibility + datum.setFinishedMaps(succeededMaps); + datum.setFinishedReduces(succeededReduces); datum.setFailedMaps(failedMaps); datum.setFailedReduces(failedReduces); + datum.setKilledMaps(killedMaps); + datum.setKilledReduces(killedReduces); datum.setMapCounters(EventWriter.toAvro(mapCounters, "MAP_COUNTERS")); datum.setReduceCounters(EventWriter.toAvro(reduceCounters, "REDUCE_COUNTERS")); @@ -102,10 +111,12 @@ public class JobFinishedEvent implements HistoryEvent { this.datum = (JobFinished) oDatum; this.jobId = JobID.forName(datum.getJobid().toString()); this.finishTime = datum.getFinishTime(); - this.finishedMaps = datum.getFinishedMaps(); - this.finishedReduces = datum.getFinishedReduces(); + this.succeededMaps = datum.getFinishedMaps(); + this.succeededReduces = datum.getFinishedReduces(); this.failedMaps = datum.getFailedMaps(); this.failedReduces = datum.getFailedReduces(); + this.killedMaps = datum.getKilledMaps(); + this.killedReduces = datum.getKilledReduces(); this.mapCounters = EventReader.fromAvro(datum.getMapCounters()); this.reduceCounters = EventReader.fromAvro(datum.getReduceCounters()); this.totalCounters = EventReader.fromAvro(datum.getTotalCounters()); @@ -120,13 +131,17 @@ public class JobFinishedEvent implements HistoryEvent { /** Get the job finish time */ public long getFinishTime() { return finishTime; } /** Get the number of finished maps for the job */ - public int getFinishedMaps() { return finishedMaps; } + public int getSucceededMaps() { return succeededMaps; } /** Get the number of finished reducers for the job */ - public int getFinishedReduces() { return finishedReduces; } + public int getSucceededReduces() { return succeededReduces; } /** Get the number of failed maps for the job */ public int getFailedMaps() { return failedMaps; } /** Get the number of failed reducers for the job */ public int getFailedReduces() { return failedReduces; } + /** Get the number of killed maps */ + public int getKilledMaps() { return killedMaps; } + /** Get the number of killed reduces */ + public int getKilledReduces() { return killedReduces; } /** Get the counters for the job */ public Counters getTotalCounters() { return totalCounters; @@ -145,12 +160,16 @@ public class JobFinishedEvent implements HistoryEvent { TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(StringUtils.toUpperCase(getEventType().name())); tEvent.addInfo("FINISH_TIME", getFinishTime()); - tEvent.addInfo("NUM_MAPS", getFinishedMaps()); - tEvent.addInfo("NUM_REDUCES", getFinishedReduces()); + tEvent.addInfo("NUM_MAPS", getSucceededMaps() + getFailedMaps() + + getKilledMaps()); + tEvent.addInfo("NUM_REDUCES", getSucceededReduces() + getFailedReduces() + + getKilledReduces()); tEvent.addInfo("FAILED_MAPS", getFailedMaps()); tEvent.addInfo("FAILED_REDUCES", getFailedReduces()); - tEvent.addInfo("FINISHED_MAPS", getFinishedMaps()); - tEvent.addInfo("FINISHED_REDUCES", getFinishedReduces()); + tEvent.addInfo("SUCCESSFUL_MAPS", getSucceededMaps()); + tEvent.addInfo("SUCCESSFUL_REDUCES", getSucceededReduces()); + tEvent.addInfo("KILLED_MAPS", getKilledMaps()); + tEvent.addInfo("KILLED_REDUCES", getKilledReduces()); // TODO replace SUCCEEDED with JobState.SUCCEEDED.toString() tEvent.addInfo("JOB_STATUS", "SUCCEEDED"); return tEvent; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java index 07699fd..28fcc92 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java @@ -376,18 +376,24 @@ public class JobHistoryParser implements HistoryEventHandler { private void handleJobFailedEvent(JobUnsuccessfulCompletionEvent event) { info.finishTime = event.getFinishTime(); - info.finishedMaps = event.getFinishedMaps(); - info.finishedReduces = event.getFinishedReduces(); + info.succeededMaps = event.getSucceededMaps(); + info.succeededReduces = event.getSucceededReduces(); + info.failedMaps = event.getFailedMaps(); + info.failedReduces = event.getFailedReduces(); + info.killedMaps = event.getKilledMaps(); + info.killedReduces = event.getKilledReduces(); info.jobStatus = StringInterner.weakIntern(event.getStatus()); info.errorInfo = StringInterner.weakIntern(event.getDiagnostics()); } private void handleJobFinishedEvent(JobFinishedEvent event) { info.finishTime = event.getFinishTime(); - info.finishedMaps = event.getFinishedMaps(); - info.finishedReduces = event.getFinishedReduces(); + info.succeededMaps = event.getSucceededMaps(); + info.succeededReduces = event.getSucceededReduces(); info.failedMaps = event.getFailedMaps(); info.failedReduces = event.getFailedReduces(); + info.killedMaps = event.getKilledMaps(); + info.killedReduces = event.getKilledReduces(); info.totalCounters = event.getTotalCounters(); info.mapCounters = event.getMapCounters(); info.reduceCounters = event.getReduceCounters(); @@ -456,8 +462,10 @@ public class JobHistoryParser implements HistoryEventHandler { int totalReduces; int failedMaps; int failedReduces; - int finishedMaps; - int finishedReduces; + int succeededMaps; + int succeededReduces; + int killedMaps; + int killedReduces; String jobStatus; Counters totalCounters; Counters mapCounters; @@ -477,7 +485,7 @@ public class JobHistoryParser implements HistoryEventHandler { public JobInfo() { submitTime = launchTime = finishTime = -1; totalMaps = totalReduces = failedMaps = failedReduces = 0; - finishedMaps = finishedReduces = 0; + succeededMaps = succeededReduces = 0; username = jobname = jobConfPath = jobQueueName = ""; tasksMap = new HashMap<TaskID, TaskInfo>(); completedTaskAttemptsMap = new HashMap<TaskAttemptID, TaskAttemptInfo>(); @@ -540,10 +548,14 @@ public class JobHistoryParser implements HistoryEventHandler { public long getFailedMaps() { return failedMaps; } /** @return the number of failed reduces */ public long getFailedReduces() { return failedReduces; } - /** @return the number of finished maps */ - public long getFinishedMaps() { return finishedMaps; } - /** @return the number of finished reduces */ - public long getFinishedReduces() { return finishedReduces; } + /** @return the number of killed maps */ + public long getKilledMaps() { return killedMaps; } + /** @return the number of killed reduces */ + public long getKilledReduces() { return killedReduces; } + /** @return the number of succeeded maps */ + public long getSucceededMaps() { return succeededMaps; } + /** @return the number of succeeded reduces */ + public long getSucceededReduces() { return succeededReduces; } /** @return the job status */ public String getJobStatus() { return jobStatus; } public String getErrorInfo() { return errorInfo; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java index ce6fa32..da31591 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java @@ -49,34 +49,58 @@ public class JobUnsuccessfulCompletionEvent implements HistoryEvent { * Create an event to record unsuccessful completion (killed/failed) of jobs * @param id Job ID * @param finishTime Finish time of the job - * @param finishedMaps Number of finished maps - * @param finishedReduces Number of finished reduces + * @param succeededMaps Number of succeeded maps + * @param succeededReduces Number of succeeded reduces + * @param failedMaps Number of failed maps + * @param failedReduces Number of failed reduces + * @param killedMaps Number of killed maps + * @param killedReduces Number of killed reduces * @param status Status of the job */ public JobUnsuccessfulCompletionEvent(JobID id, long finishTime, - int finishedMaps, - int finishedReduces, String status) { - this(id, finishTime, finishedMaps, finishedReduces, status, NODIAGS_LIST); + int succeededMaps, + int succeededReduces, + int failedMaps, + int failedReduces, + int killedMaps, + int killedReduces, + String status) { + this(id, finishTime, succeededMaps, succeededReduces, failedMaps, + failedReduces, killedMaps, killedReduces, status, NODIAGS_LIST); } /** * Create an event to record unsuccessful completion (killed/failed) of jobs * @param id Job ID * @param finishTime Finish time of the job - * @param finishedMaps Number of finished maps - * @param finishedReduces Number of finished reduces + * @param succeededMaps Number of finished maps + * @param succeededReduces Number of finished reduces + * @param failedMaps Number of failed maps + * @param failedReduces Number of failed reduces + * @param killedMaps Number of killed maps + * @param killedReduces Number of killed reduces * @param status Status of the job * @param diagnostics job runtime diagnostics */ public JobUnsuccessfulCompletionEvent(JobID id, long finishTime, - int finishedMaps, - int finishedReduces, + int succeededMaps, + int succeededReduces, + int failedMaps, + int failedReduces, + int killedMaps, + int killedReduces, String status, Iterable<String> diagnostics) { datum.setJobid(new Utf8(id.toString())); datum.setFinishTime(finishTime); - datum.setFinishedMaps(finishedMaps); - datum.setFinishedReduces(finishedReduces); + // using finishedMaps & finishedReduces in the Avro schema for backward + // compatibility + datum.setFinishedMaps(succeededMaps); + datum.setFinishedReduces(succeededReduces); + datum.setFailedMaps(failedMaps); + datum.setFailedReduces(failedReduces); + datum.setKilledMaps(killedMaps); + datum.setKilledReduces(killedReduces); datum.setJobStatus(new Utf8(status)); if (diagnostics == null) { diagnostics = NODIAGS_LIST; @@ -98,10 +122,19 @@ public class JobUnsuccessfulCompletionEvent implements HistoryEvent { } /** Get the job finish time */ public long getFinishTime() { return datum.getFinishTime(); } - /** Get the number of finished maps */ - public int getFinishedMaps() { return datum.getFinishedMaps(); } - /** Get the number of finished reduces */ - public int getFinishedReduces() { return datum.getFinishedReduces(); } + /** Get the number of succeeded maps */ + public int getSucceededMaps() { return datum.getFinishedMaps(); } + /** Get the number of succeeded reduces */ + public int getSucceededReduces() { return datum.getFinishedReduces(); } + /** Get the number of failed maps */ + public int getFailedMaps() { return datum.getFailedMaps(); } + /** Get the number of failed reduces */ + public int getFailedReduces() { return datum.getFailedReduces(); } + /** Get the number of killed maps */ + public int getKilledMaps() { return datum.getKilledMaps(); } + /** Get the number of killed reduces */ + public int getKilledReduces() { return datum.getKilledReduces(); } + /** Get the status */ public String getStatus() { return datum.getJobStatus().toString(); } /** Get the event type */ @@ -129,12 +162,19 @@ public class JobUnsuccessfulCompletionEvent implements HistoryEvent { TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(StringUtils.toUpperCase(getEventType().name())); tEvent.addInfo("FINISH_TIME", getFinishTime()); - tEvent.addInfo("NUM_MAPS", getFinishedMaps()); - tEvent.addInfo("NUM_REDUCES", getFinishedReduces()); + tEvent.addInfo("NUM_MAPS", getSucceededMaps() + getFailedMaps() + + getKilledMaps()); + tEvent.addInfo("NUM_REDUCES", getSucceededReduces() + getFailedReduces() + + getKilledReduces()); tEvent.addInfo("JOB_STATUS", getStatus()); tEvent.addInfo("DIAGNOSTICS", getDiagnostics()); - tEvent.addInfo("FINISHED_MAPS", getFinishedMaps()); - tEvent.addInfo("FINISHED_REDUCES", getFinishedReduces()); + tEvent.addInfo("SUCCESSFUL_MAPS", getSucceededMaps()); + tEvent.addInfo("SUCCESSFUL_REDUCES", getSucceededReduces()); + tEvent.addInfo("FAILED_MAPS", getFailedMaps()); + tEvent.addInfo("FAILED_REDUCES", getFailedReduces()); + tEvent.addInfo("KILLED_MAPS", getKilledMaps()); + tEvent.addInfo("KILLED_REDUCES", getKilledReduces()); + return tEvent; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java index 358e9b2..2e2dbe1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java @@ -883,8 +883,8 @@ public class TestHistoryViewerPrinter { job.totalReduces = 1; job.failedMaps = 1; job.failedReduces = 0; - job.finishedMaps = 5; - job.finishedReduces = 1; + job.succeededMaps = 5; + job.succeededReduces = 1; job.jobStatus = JobStatus.State.SUCCEEDED.name(); job.totalCounters = createCounters(); job.mapCounters = createCounters(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java index bbb126d..5afb645 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java @@ -56,7 +56,6 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo; -import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -71,7 +70,11 @@ import org.apache.hadoop.yarn.util.Records; * Data from job history file is loaded lazily. */ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job { - + // Backward compatibility: if the failed or killed map/reduce + // count is -1, that means the value was not recorded + // so we count it as 0 + private static final int UNDEFINED_VALUE = -1; + static final Log LOG = LogFactory.getLog(CompletedJob.class); private final Configuration conf; private final JobId jobId; //Can be picked from JobInfo with a conversion. @@ -104,12 +107,36 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job @Override public int getCompletedMaps() { - return (int) jobInfo.getFinishedMaps(); + int killedMaps = (int) jobInfo.getKilledMaps(); + int failedMaps = (int) jobInfo.getFailedMaps(); + + if (killedMaps == UNDEFINED_VALUE) { + killedMaps = 0; + } + + if (failedMaps == UNDEFINED_VALUE) { + failedMaps = 0; + } + + return (int) (jobInfo.getSucceededMaps() + + killedMaps + failedMaps); } @Override public int getCompletedReduces() { - return (int) jobInfo.getFinishedReduces(); + int killedReduces = (int) jobInfo.getKilledReduces(); + int failedReduces = (int) jobInfo.getFailedReduces(); + + if (killedReduces == UNDEFINED_VALUE) { + killedReduces = 0; + } + + if (failedReduces == UNDEFINED_VALUE) { + failedReduces = 0; + } + + return (int) (jobInfo.getSucceededReduces() + + killedReduces + failedReduces); } @Override @@ -481,4 +508,24 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job throw new UnsupportedOperationException( "Can't set job's priority in history"); } + + @Override + public int getFailedMaps() { + return (int) jobInfo.getFailedMaps(); + } + + @Override + public int getFailedReduces() { + return (int) jobInfo.getFailedReduces(); + } + + @Override + public int getKilledMaps() { + return (int) jobInfo.getKilledMaps(); + } + + @Override + public int getKilledReduces() { + return (int) jobInfo.getKilledReduces(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java index b3b181c..b14f0c3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java @@ -203,4 +203,23 @@ public class PartialJob implements org.apache.hadoop.mapreduce.v2.app.job.Job { "Can't set job's priority in history"); } + @Override + public int getFailedMaps() { + return -1; + } + + @Override + public int getFailedReduces() { + return -1; + } + + @Override + public int getKilledMaps() { + return -1; + } + + @Override + public int getKilledReduces() { + return -1; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java index cea336c..ecc4945 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java @@ -208,4 +208,24 @@ public class UnparsedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job { throw new UnsupportedOperationException( "Can't set job's priority in history"); } + + @Override + public int getFailedMaps() { + return -1; + } + + @Override + public int getFailedReduces() { + return -1; + } + + @Override + public int getKilledMaps() { + return -1; + } + + @Override + public int getKilledReduces() { + return -1; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java index 84b1c6d..e881b37 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java @@ -79,9 +79,12 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType; import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo; import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory; import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo; import org.apache.hadoop.net.DNSToSwitchMapping; @@ -292,7 +295,7 @@ public class TestJobHistoryParsing { Assert.assertEquals("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps); Assert.assertEquals("incorrect finishedReduces ", numReduces, - jobInfo.getFinishedReduces()); + jobInfo.getSucceededReduces()); Assert.assertEquals("incorrect uberized ", job.isUber(), jobInfo.getUberized()); Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks(); @@ -379,7 +382,7 @@ public class TestJobHistoryParsing { private long computeFinishedMaps(JobInfo jobInfo, int numMaps, int numSuccessfulMaps) { if (numMaps == numSuccessfulMaps) { - return jobInfo.getFinishedMaps(); + return jobInfo.getSucceededMaps(); } long numFinishedMaps = 0; @@ -458,6 +461,76 @@ public class TestJobHistoryParsing { } } + @Test(timeout = 30000) + public void testHistoryParsingForKilledAndFailedAttempts() throws Exception { + MRApp app = null; + JobHistory jobHistory = null; + LOG.info("STARTING testHistoryParsingForKilledAndFailedAttempts"); + try { + Configuration conf = new Configuration(); + conf.setClass( + NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, + MyResolver.class, DNSToSwitchMapping.class); + conf.set(JHAdminConfig.MR_HS_JHIST_FORMAT, "json"); + // "CommitterEventHandler" thread could be slower in some cases, + // which might cause a failed map/reduce task to fail the job + // immediately (see JobImpl.checkJobAfterTaskCompletion()). If there are + // killed events in progress, those will not be counted. Instead, + // we allow a 50% failure rate, so the job will always succeed and kill + // events will not be ignored. + conf.setInt(MRJobConfig.MAP_FAILURES_MAX_PERCENT, 50); + conf.setInt(MRJobConfig.REDUCE_FAILURES_MAXPERCENT, 50); + RackResolver.init(conf); + app = new MRAppWithHistoryWithFailedAndKilledTask(3, 3, true, this + .getClass().getName(), true); + app.submit(conf); + Job job = app.getContext().getAllJobs().values().iterator().next(); + JobId jobId = job.getID(); + app.waitForState(job, JobState.SUCCEEDED); + + // make sure all events are flushed + app.waitForState(Service.STATE.STOPPED); + + jobHistory = new JobHistory(); + jobHistory.init(conf); + HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId); + + JobHistoryParser parser; + JobInfo jobInfo; + synchronized (fileInfo) { + Path historyFilePath = fileInfo.getHistoryFile(); + FSDataInputStream in = null; + FileContext fc = null; + try { + fc = FileContext.getFileContext(conf); + in = fc.open(fc.makeQualified(historyFilePath)); + } catch (IOException ioe) { + LOG.info("Can not open history file: " + historyFilePath, ioe); + throw (new Exception("Can not open History File")); + } + + parser = new JobHistoryParser(in); + jobInfo = parser.parse(); + } + Exception parseException = parser.getParseException(); + Assert.assertNull("Caught an expected exception " + parseException, + parseException); + + assertEquals("FailedMaps", 1, jobInfo.getFailedMaps()); + assertEquals("KilledMaps", 1, jobInfo.getKilledMaps()); + assertEquals("FailedReduces", 1, jobInfo.getFailedReduces()); + assertEquals("KilledReduces", 1, jobInfo.getKilledReduces()); + } finally { + LOG.info("FINISHED testHistoryParsingForKilledAndFailedAttempts"); + if (app != null) { + app.close(); + } + if (jobHistory != null) { + jobHistory.close(); + } + } + } + @Test(timeout = 60000) public void testCountersForFailedTask() throws Exception { LOG.info("STARTING testCountersForFailedTask"); @@ -666,6 +739,40 @@ public class TestJobHistoryParsing { } } + static class MRAppWithHistoryWithFailedAndKilledTask + extends MRAppWithHistory { + + MRAppWithHistoryWithFailedAndKilledTask(int maps, int reduces, + boolean autoComplete, String testName, boolean cleanOnStart) { + super(maps, reduces, autoComplete, testName, cleanOnStart); + } + + @Override + protected void attemptLaunched(TaskAttemptId attemptID) { + final int taskId = attemptID.getTaskId().getId(); + final TaskType taskType = attemptID.getTaskId().getTaskType(); + + // map #0 --> kill + // reduce #0 --> fail + if (taskType == TaskType.MAP && taskId == 0) { + getContext().getEventHandler().handle( + new TaskEvent(attemptID.getTaskId(), TaskEventType.T_KILL)); + } else if (taskType == TaskType.MAP && taskId == 1) { + getContext().getEventHandler().handle( + new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG)); + } else if (taskType == TaskType.REDUCE && taskId == 0) { + getContext().getEventHandler().handle( + new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG)); + } else if (taskType == TaskType.REDUCE && taskId == 1) { + getContext().getEventHandler().handle( + new TaskEvent(attemptID.getTaskId(), TaskEventType.T_KILL)); + } else { + getContext().getEventHandler().handle( + new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_DONE)); + } + } + } + static class MRAppWithHistoryWithJobKilled extends MRAppWithHistory { public MRAppWithHistoryWithJobKilled(int maps, int reduces, @@ -864,6 +971,7 @@ public class TestJobHistoryParsing { if (eventId < 5) { JobUnsuccessfulCompletionEvent juce = new JobUnsuccessfulCompletionEvent(jid, 100L, 2, 0, + 0, 0, 0, 0, "JOB_FAILED", Collections.singletonList( "Task failed: " + tids[0].toString())); return juce; @@ -907,9 +1015,9 @@ public class TestJobHistoryParsing { (new Configuration()), histPath); JobInfo jobInfo = parser.parse(); LOG.info(" job info: " + jobInfo.getJobname() + " " - + jobInfo.getFinishedMaps() + " " - + jobInfo.getTotalMaps() + " " - + jobInfo.getJobId() ) ; + + jobInfo.getSucceededMaps() + " " + + jobInfo.getTotalMaps() + " " + + jobInfo.getJobId() ) ; } /** @@ -925,7 +1033,7 @@ public class TestJobHistoryParsing { (new Configuration()), histPath); JobInfo jobInfo = parser.parse(); LOG.info(" job info: " + jobInfo.getJobname() + " " - + jobInfo.getFinishedMaps() + " " + + jobInfo.getSucceededMaps() + " " + jobInfo.getTotalMaps() + " " + jobInfo.getJobId() ); } @@ -943,7 +1051,7 @@ public class TestJobHistoryParsing { (new Configuration()), histPath); JobInfo jobInfo = parser.parse(); LOG.info(" job info: " + jobInfo.getJobname() + " " - + jobInfo.getFinishedMaps() + " " + + jobInfo.getSucceededMaps() + " " + jobInfo.getTotalMaps() + " " + jobInfo.getJobId() ) ; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java index 14961d2..867c661 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java @@ -424,5 +424,25 @@ public class TestHsWebServicesAcls { @Override public void setJobPriority(Priority priority) { } + + @Override + public int getFailedMaps() { + return mockJob.getFailedMaps(); + } + + @Override + public int getFailedReduces() { + return mockJob.getFailedReduces(); + } + + @Override + public int getKilledMaps() { + return mockJob.getKilledMaps(); + } + + @Override + public int getKilledReduces() { + return mockJob.getKilledReduces(); + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04f85f3/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java index a8497f4..ef32db4 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java @@ -223,7 +223,7 @@ public class Job20LineHistoryEventEmitter extends HistoryEventEmitter { && finishedReduces != null) { return new JobUnsuccessfulCompletionEvent(jobID, Long .parseLong(finishTime), Integer.parseInt(finishedMaps), Integer - .parseInt(finishedReduces), status); + .parseInt(finishedReduces), -1, -1, -1, -1, status); } return null; @@ -256,8 +256,8 @@ public class Job20LineHistoryEventEmitter extends HistoryEventEmitter { && finishedReduces != null) { return new JobFinishedEvent(jobID, Long.parseLong(finishTime), Integer .parseInt(finishedMaps), Integer.parseInt(finishedReduces), Integer - .parseInt(failedMaps), Integer.parseInt(failedReduces), null, null, - maybeParseCounters(counters)); + .parseInt(failedMaps), Integer.parseInt(failedReduces), -1, -1, + null, null, maybeParseCounters(counters)); } return null; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
