Author: tomwhite Date: Thu Dec 20 15:54:30 2012 New Revision: 1424546 URL: http://svn.apache.org/viewvc?rev=1424546&view=rev Log: MAPREDUCE-4806. Some private methods in JobTracker.RecoveryManager are not used anymore after MAPREDUCE-3837. Contributed by Karthik Kambatla.
Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobTracker.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1424546&r1=1424545&r2=1424546&view=diff ============================================================================== --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Thu Dec 20 15:54:30 2012 @@ -354,6 +354,9 @@ Release 1.2.0 - unreleased MAPREDUCE-4860. DelegationTokenRenewal attempts to renew token even after a job is removed. (kkambatl via tucu) + MAPREDUCE-4806. Some private methods in JobTracker.RecoveryManager are not + used anymore after MAPREDUCE-3837. (Karthik Kambatla via tomwhite) + Release 1.1.2 - Unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobTracker.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1424546&r1=1424545&r2=1424546&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobTracker.java Thu Dec 20 15:54:30 2012 @@ -1309,233 +1309,6 @@ public class JobTracker implements MRCon } return ret; } - - private JobStatusChangeEvent updateJob(JobInProgress jip, - JobHistory.JobInfo job) { - // Change the job priority - String jobpriority = job.get(Keys.JOB_PRIORITY); - JobPriority priority = JobPriority.valueOf(jobpriority); - // It's important to update this via the jobtracker's api as it will - // take care of updating the event listeners too - - try { - setJobPriority(jip.getJobID(), priority); - } catch (IOException e) { - // This will not happen. JobTracker can set jobPriority of any job - // as mrOwner has the needed permissions. - LOG.warn("Unexpected. JobTracker could not do SetJobPriority on " - + jip.getJobID() + ". " + e); - } - - // Save the previous job status - JobStatus oldStatus = (JobStatus)jip.getStatus().clone(); - - // Set the start/launch time only if there are recovered tasks - // Increment the job's restart count - jip.updateJobInfo(job.getLong(JobHistory.Keys.SUBMIT_TIME), - job.getLong(JobHistory.Keys.LAUNCH_TIME)); - - // Save the new job status - JobStatus newStatus = (JobStatus)jip.getStatus().clone(); - - return new JobStatusChangeEvent(jip, EventType.START_TIME_CHANGED, oldStatus, - newStatus); - } - - private void updateTip(TaskInProgress tip, JobHistory.Task task) { - long startTime = task.getLong(Keys.START_TIME); - if (startTime != 0) { - tip.setExecStartTime(startTime); - } - - long finishTime = task.getLong(Keys.FINISH_TIME); - // For failed tasks finish-time will be missing - if (finishTime != 0) { - tip.setExecFinishTime(finishTime); - } - - String cause = task.get(Keys.TASK_ATTEMPT_ID); - if (cause.length() > 0) { - // This means that the this is a FAILED events - TaskAttemptID id = TaskAttemptID.forName(cause); - TaskStatus status = tip.getTaskStatus(id); - synchronized (JobTracker.this) { - // This will add the tip failed event in the new log - tip.getJob().failedTask(tip, id, status.getDiagnosticInfo(), - status.getPhase(), status.getRunState(), - status.getTaskTracker()); - } - } - } - - private void createTaskAttempt(JobInProgress job, - TaskAttemptID attemptId, - JobHistory.TaskAttempt attempt) - throws UnknownHostException { - TaskID id = attemptId.getTaskID(); - String type = attempt.get(Keys.TASK_TYPE); - TaskInProgress tip = job.getTaskInProgress(id); - - // I. Get the required info - TaskStatus taskStatus = null; - String trackerName = attempt.get(Keys.TRACKER_NAME); - String trackerHostName = - JobInProgress.convertTrackerNameToHostName(trackerName); - // recover the port information. - int port = 0; // default to 0 - String hport = attempt.get(Keys.HTTP_PORT); - if (hport != null && hport.length() > 0) { - port = attempt.getInt(Keys.HTTP_PORT); - } - - long attemptStartTime = attempt.getLong(Keys.START_TIME); - - // II. Create the (appropriate) task status - if (type.equals(Values.MAP.name())) { - taskStatus = - new MapTaskStatus(attemptId, 0.0f, job.getNumSlotsPerTask(TaskType.MAP), - TaskStatus.State.RUNNING, "", "", trackerName, - TaskStatus.Phase.MAP, new Counters()); - } else { - taskStatus = - new ReduceTaskStatus(attemptId, 0.0f, job.getNumSlotsPerTask(TaskType.REDUCE), - TaskStatus.State.RUNNING, "", "", trackerName, - TaskStatus.Phase.REDUCE, new Counters()); - } - - // Set the start time - taskStatus.setStartTime(attemptStartTime); - - List<TaskStatus> ttStatusList = new ArrayList<TaskStatus>(); - ttStatusList.add(taskStatus); - - // III. Create the dummy tasktracker status - TaskTrackerStatus ttStatus = - new TaskTrackerStatus(trackerName, trackerHostName, port, ttStatusList, - 0 , 0, 0, 0); - ttStatus.setLastSeen(clock.getTime()); - - synchronized (JobTracker.this) { - synchronized (taskTrackers) { - synchronized (trackerExpiryQueue) { - // IV. Register a new tracker - TaskTracker taskTracker = getTaskTracker(trackerName); - boolean isTrackerRegistered = (taskTracker != null); - if (!isTrackerRegistered) { - markTracker(trackerName); // add the tracker to recovery-manager - taskTracker = new TaskTracker(trackerName); - taskTracker.setStatus(ttStatus); - addNewTracker(taskTracker); - } - - // V. Update the tracker status - // This will update the meta info of the jobtracker and also add the - // tracker status if missing i.e register it - updateTaskTrackerStatus(trackerName, ttStatus); - } - } - // Register the attempt with job and tip, under JobTracker lock. - // Since, as of today they are atomic through heartbeat. - // VI. Register the attempt - // a) In the job - job.addRunningTaskToTIP(tip, attemptId, ttStatus, false); - // b) In the tip - tip.updateStatus(taskStatus); - } - - // VII. Make an entry in the launched tasks - expireLaunchingTasks.addNewTask(attemptId); - } - - private void addSuccessfulAttempt(JobInProgress job, - TaskAttemptID attemptId, - JobHistory.TaskAttempt attempt) { - // I. Get the required info - TaskID taskId = attemptId.getTaskID(); - String type = attempt.get(Keys.TASK_TYPE); - - TaskInProgress tip = job.getTaskInProgress(taskId); - long attemptFinishTime = attempt.getLong(Keys.FINISH_TIME); - - // Get the task status and the tracker name and make a copy of it - TaskStatus taskStatus = (TaskStatus)tip.getTaskStatus(attemptId).clone(); - taskStatus.setFinishTime(attemptFinishTime); - - String stateString = attempt.get(Keys.STATE_STRING); - - // Update the basic values - taskStatus.setStateString(stateString); - taskStatus.setProgress(1.0f); - taskStatus.setRunState(TaskStatus.State.SUCCEEDED); - - // Set the shuffle/sort finished times - if (type.equals(Values.REDUCE.name())) { - long shuffleTime = - Long.parseLong(attempt.get(Keys.SHUFFLE_FINISHED)); - long sortTime = - Long.parseLong(attempt.get(Keys.SORT_FINISHED)); - taskStatus.setShuffleFinishTime(shuffleTime); - taskStatus.setSortFinishTime(sortTime); - } - - // Add the counters - String counterString = attempt.get(Keys.COUNTERS); - Counters counter = null; - //TODO Check if an exception should be thrown - try { - counter = Counters.fromEscapedCompactString(counterString); - } catch (ParseException pe) { - counter = new Counters(); // Set it to empty counter - } - taskStatus.setCounters(counter); - - synchronized (JobTracker.this) { - // II. Replay the status - job.updateTaskStatus(tip, taskStatus); - } - - // III. Prevent the task from expiry - expireLaunchingTasks.removeTask(attemptId); - } - - private void addUnsuccessfulAttempt(JobInProgress job, - TaskAttemptID attemptId, - JobHistory.TaskAttempt attempt) { - // I. Get the required info - TaskID taskId = attemptId.getTaskID(); - TaskInProgress tip = job.getTaskInProgress(taskId); - long attemptFinishTime = attempt.getLong(Keys.FINISH_TIME); - - TaskStatus taskStatus = (TaskStatus)tip.getTaskStatus(attemptId).clone(); - taskStatus.setFinishTime(attemptFinishTime); - - // Reset the progress - taskStatus.setProgress(0.0f); - - String stateString = attempt.get(Keys.STATE_STRING); - taskStatus.setStateString(stateString); - - boolean hasFailed = - attempt.get(Keys.TASK_STATUS).equals(Values.FAILED.name()); - // Set the state failed/killed - if (hasFailed) { - taskStatus.setRunState(TaskStatus.State.FAILED); - } else { - taskStatus.setRunState(TaskStatus.State.KILLED); - } - - // Get/Set the error msg - String diagInfo = attempt.get(Keys.ERROR); - taskStatus.setDiagnosticInfo(diagInfo); // diag info - - synchronized (JobTracker.this) { - // II. Update the task status - job.updateTaskStatus(tip, taskStatus); - } - - // III. Prevent the task from expiry - expireLaunchingTasks.removeTask(attemptId); - } Path getRestartCountFile() { return new Path(getSystemDir(), "jobtracker.info");