Author: szetszwo Date: Wed Oct 24 20:42:03 2012 New Revision: 1401869 URL: http://svn.apache.org/viewvc?rev=1401869&view=rev Log: Merge r1400738 through r1401868 from trunk.
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleanerImpl.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/CountersStrings.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/c++/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/build.xml (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/data_join/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/index/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/examples/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/java/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/webapps/job/ (props changed) Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1400738-1401868 Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt Wed Oct 24 20:42:03 2012 @@ -189,6 +189,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4736. Remove obsolete option [-rootDir] from TestDFSIO. (Brandon Li via suresh) + MAPREDUCE-4637. Handle TaskAttempt diagnostic updates while in the NEW and + UNASSIGNED states. (Mayank Bansal via sseth) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES @@ -599,6 +602,14 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4733. Reducer can fail to make progress during shuffle if too many reducers complete consecutively. (Jason Lowe via vinodkv) + MAPREDUCE-4740. only .jars can be added to the Distributed Cache + classpath. (Robert Joseph Evans via jlowe) + + MAPREDUCE-4229. Intern counter names in the JT (Miomir Boljanovic and bobby via daryn) + + MAPREDUCE-4741. WARN and ERROR messages logged during normal AM shutdown. + (Vinod Kumar Vavilapalli via jlowe) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1400738-1401868 Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Wed Oct 24 20:42:03 2012 @@ -107,6 +107,8 @@ import org.apache.hadoop.yarn.service.Co import org.apache.hadoop.yarn.service.Service; import org.apache.hadoop.yarn.util.ConverterUtils; +import com.google.common.annotations.VisibleForTesting; + /** * The Map-Reduce Application Master. * The state machine is encapsulated in the implementation of Job interface. @@ -398,52 +400,65 @@ public class MRAppMaster extends Composi protected void sysexit() { System.exit(0); } - - private class JobFinishEventHandler implements EventHandler<JobFinishEvent> { - @Override - public void handle(JobFinishEvent event) { - // job has finished - // this is the only job, so shut down the Appmaster - // note in a workflow scenario, this may lead to creation of a new - // job (FIXME?) - // Send job-end notification - if (getConfig().get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL) != null) { - try { - LOG.info("Job end notification started for jobID : " - + job.getReport().getJobId()); - JobEndNotifier notifier = new JobEndNotifier(); - notifier.setConf(getConfig()); - notifier.notify(job.getReport()); - } catch (InterruptedException ie) { - LOG.warn("Job end notification interrupted for jobID : " - + job.getReport().getJobId(), ie); - } - } - // TODO:currently just wait for some time so clients can know the - // final states. Will be removed once RM come on. + @VisibleForTesting + public void shutDownJob() { + // job has finished + // this is the only job, so shut down the Appmaster + // note in a workflow scenario, this may lead to creation of a new + // job (FIXME?) + // Send job-end notification + if (getConfig().get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL) != null) { try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); + LOG.info("Job end notification started for jobID : " + + job.getReport().getJobId()); + JobEndNotifier notifier = new JobEndNotifier(); + notifier.setConf(getConfig()); + notifier.notify(job.getReport()); + } catch (InterruptedException ie) { + LOG.warn("Job end notification interrupted for jobID : " + + job.getReport().getJobId(), ie); } + } - try { - //We are finishing cleanly so this is the last retry - isLastAMRetry = true; - // Stop all services - // This will also send the final report to the ResourceManager - LOG.info("Calling stop for all the services"); - stop(); - - } catch (Throwable t) { - LOG.warn("Graceful stop failed ", t); - } + // TODO:currently just wait for some time so clients can know the + // final states. Will be removed once RM come on. + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + try { + //We are finishing cleanly so this is the last retry + isLastAMRetry = true; + // Stop all services + // This will also send the final report to the ResourceManager + LOG.info("Calling stop for all the services"); + MRAppMaster.this.stop(); + + } catch (Throwable t) { + LOG.warn("Graceful stop failed ", t); + } - //Bring the process down by force. - //Not needed after HADOOP-7140 - LOG.info("Exiting MR AppMaster..GoodBye!"); - sysexit(); + //Bring the process down by force. + //Not needed after HADOOP-7140 + LOG.info("Exiting MR AppMaster..GoodBye!"); + sysexit(); + } + + private class JobFinishEventHandler implements EventHandler<JobFinishEvent> { + @Override + public void handle(JobFinishEvent event) { + // Create a new thread to shutdown the AM. We should not do it in-line + // to avoid blocking the dispatcher itself. + new Thread() { + + @Override + public void run() { + shutDownJob(); + } + }.start(); } } Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Oct 24 20:42:03 2012 @@ -200,6 +200,10 @@ public abstract class TaskAttemptImpl im TaskAttemptEventType.TA_KILL, new KilledTransition()) .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_FAILMSG, new FailedTransition()) + .addTransition(TaskAttemptStateInternal.NEW, + TaskAttemptStateInternal.NEW, + TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, + DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Transitions from the UNASSIGNED state. .addTransition(TaskAttemptStateInternal.UNASSIGNED, @@ -211,6 +215,10 @@ public abstract class TaskAttemptImpl im .addTransition(TaskAttemptStateInternal.UNASSIGNED, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_FAILMSG, new DeallocateContainerTransition( TaskAttemptStateInternal.FAILED, true)) + .addTransition(TaskAttemptStateInternal.UNASSIGNED, + TaskAttemptStateInternal.UNASSIGNED, + TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, + DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Transitions from the ASSIGNED state. .addTransition(TaskAttemptStateInternal.ASSIGNED, TaskAttemptStateInternal.RUNNING, Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java Wed Oct 24 20:42:03 2012 @@ -30,6 +30,7 @@ import java.util.concurrent.LinkedBlocki import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -81,6 +82,7 @@ public class ContainerLauncherImpl exten protected BlockingQueue<ContainerLauncherEvent> eventQueue = new LinkedBlockingQueue<ContainerLauncherEvent>(); YarnRPC rpc; + private final AtomicBoolean stopped; private Container getContainer(ContainerLauncherEvent event) { ContainerId id = event.getContainerID(); @@ -237,6 +239,7 @@ public class ContainerLauncherImpl exten public ContainerLauncherImpl(AppContext context) { super(ContainerLauncherImpl.class.getName()); this.context = context; + this.stopped = new AtomicBoolean(false); } @Override @@ -271,11 +274,13 @@ public class ContainerLauncherImpl exten @Override public void run() { ContainerLauncherEvent event = null; - while (!Thread.currentThread().isInterrupted()) { + while (!stopped.get() && !Thread.currentThread().isInterrupted()) { try { event = eventQueue.take(); } catch (InterruptedException e) { - LOG.error("Returning, interrupted : " + e); + if (!stopped.get()) { + LOG.error("Returning, interrupted : " + e); + } return; } int poolSize = launcherPool.getCorePoolSize(); @@ -324,6 +329,10 @@ public class ContainerLauncherImpl exten } public void stop() { + if (stopped.getAndSet(true)) { + // return if already stopped + return; + } // shutdown any containers that might be left running shutdownAllContainers(); eventHandlingThread.interrupt(); Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java Wed Oct 24 20:42:03 2012 @@ -67,7 +67,7 @@ public abstract class RMCommunicator ext private int rmPollInterval;//millis protected ApplicationId applicationId; protected ApplicationAttemptId applicationAttemptId; - private AtomicBoolean stopped; + private final AtomicBoolean stopped; protected Thread allocatorThread; @SuppressWarnings("rawtypes") protected EventHandler eventHandler; @@ -239,7 +239,9 @@ public abstract class RMCommunicator ext // TODO: for other exceptions } } catch (InterruptedException e) { - LOG.warn("Allocated thread interrupted. Returning."); + if (!stopped.get()) { + LOG.warn("Allocated thread interrupted. Returning."); + } return; } } Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Wed Oct 24 20:42:03 2012 @@ -32,6 +32,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -84,7 +85,7 @@ public class RMContainerAllocator extend private static final Priority PRIORITY_MAP; private Thread eventHandlingThread; - private volatile boolean stopEventHandling; + private final AtomicBoolean stopped; static { PRIORITY_FAST_FAIL_MAP = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Priority.class); @@ -145,6 +146,7 @@ public class RMContainerAllocator extend public RMContainerAllocator(ClientService clientService, AppContext context) { super(clientService, context); + this.stopped = new AtomicBoolean(false); } @Override @@ -176,11 +178,13 @@ public class RMContainerAllocator extend ContainerAllocatorEvent event; - while (!stopEventHandling && !Thread.currentThread().isInterrupted()) { + while (!stopped.get() && !Thread.currentThread().isInterrupted()) { try { event = RMContainerAllocator.this.eventQueue.take(); } catch (InterruptedException e) { - LOG.error("Returning, interrupted : " + e); + if (!stopped.get()) { + LOG.error("Returning, interrupted : " + e); + } return; } @@ -234,7 +238,10 @@ public class RMContainerAllocator extend @Override public void stop() { - this.stopEventHandling = true; + if (stopped.getAndSet(true)) { + // return if already stopped + return; + } eventHandlingThread.interrupt(); super.stop(); LOG.info("Final Stats: " + getStat()); Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleanerImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleanerImpl.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleanerImpl.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleanerImpl.java Wed Oct 24 20:42:03 2012 @@ -23,6 +23,7 @@ import java.util.concurrent.LinkedBlocki import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -43,10 +44,12 @@ public class TaskCleanerImpl extends Abs private Thread eventHandlingThread; private BlockingQueue<TaskCleanupEvent> eventQueue = new LinkedBlockingQueue<TaskCleanupEvent>(); + private final AtomicBoolean stopped; public TaskCleanerImpl(AppContext context) { super("TaskCleaner"); this.context = context; + this.stopped = new AtomicBoolean(false); } public void start() { @@ -59,11 +62,13 @@ public class TaskCleanerImpl extends Abs @Override public void run() { TaskCleanupEvent event = null; - while (!Thread.currentThread().isInterrupted()) { + while (!stopped.get() && !Thread.currentThread().isInterrupted()) { try { event = eventQueue.take(); } catch (InterruptedException e) { - LOG.error("Returning, interrupted : " + e); + if (!stopped.get()) { + LOG.error("Returning, interrupted : " + e); + } return; } // the events from the queue are handled in parallel @@ -77,6 +82,10 @@ public class TaskCleanerImpl extends Abs } public void stop() { + if (stopped.getAndSet(true)) { + // return if already stopped + return; + } eventHandlingThread.interrupt(); launcherPool.shutdown(); super.stop(); Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java Wed Oct 24 20:42:03 2012 @@ -21,17 +21,15 @@ package org.apache.hadoop.mapreduce.v2.a import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.times; import java.io.IOException; import junit.framework.Assert; import junit.framework.TestCase; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -49,7 +47,6 @@ import org.apache.hadoop.yarn.YarnExcept import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.service.AbstractService; @@ -68,7 +65,6 @@ import org.junit.Test; private Path stagingJobPath = new Path(stagingJobDir); private final static RecordFactory recordFactory = RecordFactoryProvider. getRecordFactory(null); - private static final Log LOG = LogFactory.getLog(TestStagingCleanup.class); @Test public void testDeletionofStaging() throws IOException { @@ -86,9 +82,7 @@ import org.junit.Test; jobid.setAppId(appId); MRAppMaster appMaster = new TestMRApp(attemptId); appMaster.init(conf); - EventHandler<JobFinishEvent> handler = - appMaster.createJobFinishEventHandler(); - handler.handle(new JobFinishEvent(jobid)); + appMaster.shutDownJob(); verify(fs).delete(stagingJobPath, true); } Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java Wed Oct 24 20:42:03 2012 @@ -546,6 +546,105 @@ public class TestTaskAttempt{ eventHandler.internalError); } + @Test + public void testAppDiognosticEventOnUnassignedTask() throws Exception { + ApplicationId appId = BuilderUtils.newApplicationId(1, 2); + ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + appId, 0); + JobId jobId = MRBuilderUtils.newJobId(appId, 1); + TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP); + TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0); + Path jobFile = mock(Path.class); + + MockEventHandler eventHandler = new MockEventHandler(); + TaskAttemptListener taListener = mock(TaskAttemptListener.class); + when(taListener.getAddress()).thenReturn( + new InetSocketAddress("localhost", 0)); + + JobConf jobConf = new JobConf(); + jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); + jobConf.setBoolean("fs.file.impl.disable.cache", true); + jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, ""); + jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10"); + + TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class); + when(splits.getLocations()).thenReturn(new String[] { "127.0.0.1" }); + + AppContext appCtx = mock(AppContext.class); + ClusterInfo clusterInfo = mock(ClusterInfo.class); + Resource resource = mock(Resource.class); + when(appCtx.getClusterInfo()).thenReturn(clusterInfo); + when(clusterInfo.getMinContainerCapability()).thenReturn(resource); + when(resource.getMemory()).thenReturn(1024); + + TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, + jobFile, 1, splits, jobConf, taListener, mock(OutputCommitter.class), + mock(Token.class), new Credentials(), new SystemClock(), appCtx); + + NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0); + ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3); + Container container = mock(Container.class); + when(container.getId()).thenReturn(contId); + when(container.getNodeId()).thenReturn(nid); + when(container.getNodeHttpAddress()).thenReturn("localhost:0"); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_SCHEDULE)); + taImpl.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptId, + "Task got killed")); + assertFalse( + "InternalError occurred trying to handle TA_DIAGNOSTICS_UPDATE on assigned task", + eventHandler.internalError); + } + + @Test + public void testAppDiognosticEventOnNewTask() throws Exception { + ApplicationId appId = BuilderUtils.newApplicationId(1, 2); + ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + appId, 0); + JobId jobId = MRBuilderUtils.newJobId(appId, 1); + TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP); + TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0); + Path jobFile = mock(Path.class); + + MockEventHandler eventHandler = new MockEventHandler(); + TaskAttemptListener taListener = mock(TaskAttemptListener.class); + when(taListener.getAddress()).thenReturn( + new InetSocketAddress("localhost", 0)); + + JobConf jobConf = new JobConf(); + jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); + jobConf.setBoolean("fs.file.impl.disable.cache", true); + jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, ""); + jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10"); + + TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class); + when(splits.getLocations()).thenReturn(new String[] { "127.0.0.1" }); + + AppContext appCtx = mock(AppContext.class); + ClusterInfo clusterInfo = mock(ClusterInfo.class); + Resource resource = mock(Resource.class); + when(appCtx.getClusterInfo()).thenReturn(clusterInfo); + when(clusterInfo.getMinContainerCapability()).thenReturn(resource); + when(resource.getMemory()).thenReturn(1024); + + TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, + jobFile, 1, splits, jobConf, taListener, mock(OutputCommitter.class), + mock(Token.class), new Credentials(), new SystemClock(), appCtx); + + NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0); + ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3); + Container container = mock(Container.class); + when(container.getId()).thenReturn(contId); + when(container.getNodeId()).thenReturn(nid); + when(container.getNodeHttpAddress()).thenReturn("localhost:0"); + taImpl.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptId, + "Task got killed")); + assertFalse( + "InternalError occurred trying to handle TA_DIAGNOSTICS_UPDATE on assigned task", + eventHandler.internalError); + } + + public static class MockEventHandler implements EventHandler { public boolean internalError; Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Wed Oct 24 20:42:03 2012 @@ -191,6 +191,7 @@ public class MRApps extends Apps { // TODO: Remove duplicates. } + @SuppressWarnings("deprecation") public static void setClasspath(Map<String, String> environment, Configuration conf) throws IOException { boolean userClassesTakesPrecedence = @@ -218,11 +219,66 @@ public class MRApps extends Apps { environment, Environment.CLASSPATH.name(), Environment.PWD.$() + Path.SEPARATOR + "*"); + // a * in the classpath will only find a .jar, so we need to filter out + // all .jars and add everything else + addToClasspathIfNotJar(DistributedCache.getFileClassPaths(conf), + DistributedCache.getCacheFiles(conf), + conf, + environment); + addToClasspathIfNotJar(DistributedCache.getArchiveClassPaths(conf), + DistributedCache.getCacheArchives(conf), + conf, + environment); if (userClassesTakesPrecedence) { MRApps.setMRFrameworkClasspath(environment, conf); } } + /** + * Add the paths to the classpath if they are not jars + * @param paths the paths to add to the classpath + * @param withLinks the corresponding paths that may have a link name in them + * @param conf used to resolve the paths + * @param environment the environment to update CLASSPATH in + * @throws IOException if there is an error resolving any of the paths. + */ + private static void addToClasspathIfNotJar(Path[] paths, + URI[] withLinks, Configuration conf, + Map<String, String> environment) throws IOException { + if (paths != null) { + HashMap<Path, String> linkLookup = new HashMap<Path, String>(); + if (withLinks != null) { + for (URI u: withLinks) { + Path p = new Path(u); + FileSystem remoteFS = p.getFileSystem(conf); + p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), + remoteFS.getWorkingDirectory())); + String name = (null == u.getFragment()) + ? p.getName() : u.getFragment(); + if (!name.toLowerCase().endsWith(".jar")) { + linkLookup.put(p, name); + } + } + } + + for (Path p : paths) { + FileSystem remoteFS = p.getFileSystem(conf); + p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), + remoteFS.getWorkingDirectory())); + String name = linkLookup.get(p); + if (name == null) { + name = p.getName(); + } + if(!name.toLowerCase().endsWith(".jar")) { + Apps.addToEnvironment( + environment, + Environment.CLASSPATH.name(), + Environment.PWD.$() + Path.SEPARATOR + name); + } + } + } + } + private static final String STAGING_CONSTANT = ".staging"; public static Path getStagingAreaDir(Configuration conf, String user) { return new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR, @@ -261,8 +317,7 @@ public class MRApps extends Apps { DistributedCache.getCacheArchives(conf), parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)), getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), - DistributedCache.getArchiveVisibilities(conf), - DistributedCache.getArchiveClassPaths(conf)); + DistributedCache.getArchiveVisibilities(conf)); // Cache files parseDistributedCacheArtifacts(conf, @@ -271,8 +326,7 @@ public class MRApps extends Apps { DistributedCache.getCacheFiles(conf), parseTimeStamps(DistributedCache.getFileTimestamps(conf)), getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES), - DistributedCache.getFileVisibilities(conf), - DistributedCache.getFileClassPaths(conf)); + DistributedCache.getFileVisibilities(conf)); } private static String getResourceDescription(LocalResourceType type) { @@ -289,8 +343,8 @@ public class MRApps extends Apps { Configuration conf, Map<String, LocalResource> localResources, LocalResourceType type, - URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[], - Path[] pathsToPutOnClasspath) throws IOException { + URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[]) + throws IOException { if (uris != null) { // Sanity check @@ -304,15 +358,6 @@ public class MRApps extends Apps { ); } - Map<String, Path> classPaths = new HashMap<String, Path>(); - if (pathsToPutOnClasspath != null) { - for (Path p : pathsToPutOnClasspath) { - FileSystem remoteFS = p.getFileSystem(conf); - p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), - remoteFS.getWorkingDirectory())); - classPaths.put(p.toUri().getPath().toString(), p); - } - } for (int i = 0; i < uris.length; ++i) { URI u = uris[i]; Path p = new Path(u); Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Wed Oct 24 20:42:03 2012 @@ -18,6 +18,8 @@ package org.apache.hadoop.mapreduce.v2.util; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.util.HashMap; @@ -42,12 +44,36 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; - +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import static org.junit.Assert.*; import static org.mockito.Mockito.*; public class TestMRApps { + private static File testWorkDir = null; + + @BeforeClass + public static void setupTestDirs() throws IOException { + testWorkDir = new File("target", TestMRApps.class.getCanonicalName()); + delete(testWorkDir); + testWorkDir.mkdirs(); + testWorkDir = testWorkDir.getAbsoluteFile(); + } + + @AfterClass + public static void cleanupTestDirs() throws IOException { + if (testWorkDir != null) { + delete(testWorkDir); + } + } + + private static void delete(File dir) throws IOException { + Path p = new Path("file://"+dir.getAbsolutePath()); + Configuration conf = new Configuration(); + FileSystem fs = p.getFileSystem(conf); + fs.delete(p, true); + } @Test public void testJobIDtoString() { JobId jid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class); @@ -154,6 +180,28 @@ public class TestMRApps { } assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath)); } + + @Test public void testSetClasspathWithArchives () throws IOException { + File testTGZ = new File(testWorkDir, "test.tgz"); + FileOutputStream out = new FileOutputStream(testTGZ); + out.write(0); + out.close(); + Job job = Job.getInstance(); + Configuration conf = job.getConfiguration(); + conf.set(MRJobConfig.CLASSPATH_ARCHIVES, "file://" + + testTGZ.getAbsolutePath()); + conf.set(MRJobConfig.CACHE_ARCHIVES, "file://" + + testTGZ.getAbsolutePath() + "#testTGZ"); + Map<String, String> environment = new HashMap<String, String>(); + MRApps.setClasspath(environment, conf); + assertTrue(environment.get("CLASSPATH").startsWith("$PWD:")); + String confClasspath = job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH); + if (confClasspath != null) { + confClasspath = confClasspath.replaceAll(",\\s*", ":").trim(); + } + assertTrue(environment.get("CLASSPATH").contains(confClasspath)); + assertTrue(environment.get("CLASSPATH").contains("testTGZ")); + } @Test public void testSetClasspathWithUserPrecendence() { Configuration conf = new Configuration(); Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java Wed Oct 24 20:42:03 2012 @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.CounterGroup; import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.util.StringInterner; import org.apache.avro.Schema; import org.apache.avro.io.Decoder; @@ -170,9 +171,11 @@ public class EventReader implements Clos Counters result = new Counters(); for (JhCounterGroup g : counters.groups) { CounterGroup group = - result.addGroup(g.name.toString(), g.displayName.toString()); + result.addGroup(StringInterner.weakIntern(g.name.toString()), + StringInterner.weakIntern(g.displayName.toString())); for (JhCounter c : g.counts) { - group.addCounter(c.name.toString(), c.displayName.toString(), c.value); + group.addCounter(StringInterner.weakIntern(c.name.toString()), + StringInterner.weakIntern(c.displayName.toString()), c.value); } } return result; Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java Wed Oct 24 20:42:03 2012 @@ -42,6 +42,7 @@ import org.apache.hadoop.mapreduce.TaskI import org.apache.hadoop.mapred.TaskStatus; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -226,10 +227,10 @@ public class JobHistoryParser { TaskAttemptInfo attemptInfo = taskInfo.attemptsMap.get(event.getAttemptId()); attemptInfo.finishTime = event.getFinishTime(); - attemptInfo.status = event.getTaskStatus(); - attemptInfo.state = event.getState(); + attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus()); + attemptInfo.state = StringInterner.weakIntern(event.getState()); attemptInfo.counters = event.getCounters(); - attemptInfo.hostname = event.getHostname(); + attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); } private void handleReduceAttemptFinishedEvent @@ -238,14 +239,14 @@ public class JobHistoryParser { TaskAttemptInfo attemptInfo = taskInfo.attemptsMap.get(event.getAttemptId()); attemptInfo.finishTime = event.getFinishTime(); - attemptInfo.status = event.getTaskStatus(); - attemptInfo.state = event.getState(); + attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus()); + attemptInfo.state = StringInterner.weakIntern(event.getState()); attemptInfo.shuffleFinishTime = event.getShuffleFinishTime(); attemptInfo.sortFinishTime = event.getSortFinishTime(); attemptInfo.counters = event.getCounters(); - attemptInfo.hostname = event.getHostname(); + attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); attemptInfo.port = event.getPort(); - attemptInfo.rackname = event.getRackName(); + attemptInfo.rackname = StringInterner.weakIntern(event.getRackName()); } private void handleMapAttemptFinishedEvent(MapAttemptFinishedEvent event) { @@ -253,13 +254,13 @@ public class JobHistoryParser { TaskAttemptInfo attemptInfo = taskInfo.attemptsMap.get(event.getAttemptId()); attemptInfo.finishTime = event.getFinishTime(); - attemptInfo.status = event.getTaskStatus(); - attemptInfo.state = event.getState(); + attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus()); + attemptInfo.state = StringInterner.weakIntern(event.getState()); attemptInfo.mapFinishTime = event.getMapFinishTime(); attemptInfo.counters = event.getCounters(); - attemptInfo.hostname = event.getHostname(); + attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); attemptInfo.port = event.getPort(); - attemptInfo.rackname = event.getRackName(); + attemptInfo.rackname = StringInterner.weakIntern(event.getRackName()); } private void handleTaskAttemptFailedEvent( @@ -269,10 +270,10 @@ public class JobHistoryParser { taskInfo.attemptsMap.get(event.getTaskAttemptId()); attemptInfo.finishTime = event.getFinishTime(); attemptInfo.error = event.getError(); - attemptInfo.status = event.getTaskStatus(); - attemptInfo.hostname = event.getHostname(); + attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus()); + attemptInfo.hostname = StringInterner.weakIntern(event.getHostname()); attemptInfo.port = event.getPort(); - attemptInfo.rackname = event.getRackName(); + attemptInfo.rackname = StringInterner.weakIntern(event.getRackName()); attemptInfo.shuffleFinishTime = event.getFinishTime(); attemptInfo.sortFinishTime = event.getFinishTime(); attemptInfo.mapFinishTime = event.getFinishTime(); @@ -300,7 +301,7 @@ public class JobHistoryParser { attemptInfo.startTime = event.getStartTime(); attemptInfo.attemptId = event.getTaskAttemptId(); attemptInfo.httpPort = event.getHttpPort(); - attemptInfo.trackerName = event.getTrackerName(); + attemptInfo.trackerName = StringInterner.weakIntern(event.getTrackerName()); attemptInfo.taskType = event.getTaskType(); attemptInfo.shufflePort = event.getShufflePort(); attemptInfo.containerId = event.getContainerId(); @@ -344,7 +345,7 @@ public class JobHistoryParser { info.finishTime = event.getFinishTime(); info.finishedMaps = event.getFinishedMaps(); info.finishedReduces = event.getFinishedReduces(); - info.jobStatus = event.getStatus(); + info.jobStatus = StringInterner.weakIntern(event.getStatus()); } private void handleJobFinishedEvent(JobFinishedEvent event) { @@ -375,7 +376,7 @@ public class JobHistoryParser { amInfo.appAttemptId = event.getAppAttemptId(); amInfo.startTime = event.getStartTime(); amInfo.containerId = event.getContainerId(); - amInfo.nodeManagerHost = event.getNodeManagerHost(); + amInfo.nodeManagerHost = StringInterner.weakIntern(event.getNodeManagerHost()); amInfo.nodeManagerPort = event.getNodeManagerPort(); amInfo.nodeManagerHttpPort = event.getNodeManagerHttpPort(); if (info.amInfos == null) { @@ -393,11 +394,11 @@ public class JobHistoryParser { private void handleJobSubmittedEvent(JobSubmittedEvent event) { info.jobid = event.getJobId(); info.jobname = event.getJobName(); - info.username = event.getUserName(); + info.username = StringInterner.weakIntern(event.getUserName()); info.submitTime = event.getSubmitTime(); info.jobConfPath = event.getJobConfPath(); info.jobACLs = event.getJobAcls(); - info.jobQueueName = event.getJobQueueName(); + info.jobQueueName = StringInterner.weakIntern(event.getJobQueueName()); } /** Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/CountersStrings.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/CountersStrings.java?rev=1401869&r1=1401868&r2=1401869&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/CountersStrings.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/CountersStrings.java Wed Oct 24 20:42:03 2012 @@ -28,6 +28,7 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.mapreduce.counters.AbstractCounters; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.counters.CounterGroupBase; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.util.StringUtils; /** @@ -235,13 +236,13 @@ public class CountersStrings { // Get the actual name String groupName = - getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex); - groupName = unescape(groupName); + StringInterner.weakIntern(getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex)); + groupName = StringInterner.weakIntern(unescape(groupName)); // Get the display name String groupDisplayName = - getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex); - groupDisplayName = unescape(groupDisplayName); + StringInterner.weakIntern(getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex)); + groupDisplayName = StringInterner.weakIntern(unescape(groupDisplayName)); // Get the counters G group = counters.getGroup(groupName); @@ -255,13 +256,13 @@ public class CountersStrings { // Get the actual name String counterName = - getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex); - counterName = unescape(counterName); + StringInterner.weakIntern(getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex)); + counterName = StringInterner.weakIntern(unescape(counterName)); // Get the display name String counterDisplayName = - getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex); - counterDisplayName = unescape(counterDisplayName); + StringInterner.weakIntern(getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex)); + counterDisplayName = StringInterner.weakIntern(unescape(counterDisplayName)); // Get the value long value = Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/c++/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/c++:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/block_forensics/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/build-contrib.xml ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/build.xml ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/data_join/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/index/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/contrib/vaidya/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/examples/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1400738-1401868 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/src/webapps/job/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1400738-1401868