Author: daryn Date: Tue Jul 17 19:32:14 2012 New Revision: 1362618 URL: http://svn.apache.org/viewvc?rev=1362618&view=rev Log: MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't init. (Jason Lowe via daryn)
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1362618&r1=1362617&r2=1362618&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 17 19:32:14 2012 @@ -724,6 +724,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4283. Display tail of aggregated logs by default (Jason Lowe via bobby) + MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't + init. (Jason Lowe via daryn) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1362618&r1=1362617&r2=1362618&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Tue Jul 17 19:32:14 2012 @@ -342,14 +342,14 @@ public class LogAggregationService exten // A container is complete. Put this containers' logs up for aggregation if // this containers' logs are needed. - if (!this.appLogAggregators.containsKey( - containerId.getApplicationAttemptId().getApplicationId())) { - throw new YarnException("Application is not initialized yet for " - + containerId); + AppLogAggregator aggregator = this.appLogAggregators.get( + containerId.getApplicationAttemptId().getApplicationId()); + if (aggregator == null) { + LOG.warn("Log aggregation is not initialized for " + containerId + + ", did it fail to start?"); + return; } - this.appLogAggregators.get( - containerId.getApplicationAttemptId().getApplicationId()) - .startContainerLogAggregation(containerId, exitCode == 0); + aggregator.startContainerLogAggregation(containerId, exitCode == 0); } private void stopApp(ApplicationId appId) { @@ -357,11 +357,13 @@ public class LogAggregationService exten // App is complete. Finish up any containers' pending log aggregation and // close the application specific logFile. - if (!this.appLogAggregators.containsKey(appId)) { - throw new YarnException("Application is not initialized yet for " - + appId); + AppLogAggregator aggregator = this.appLogAggregators.get(appId); + if (aggregator == null) { + LOG.warn("Log aggregation is not initialized for " + appId + + ", did it fail to start?"); + return; } - this.appLogAggregators.get(appId).finishLogAggregation(); + aggregator.finishLogAggregation(); } @Override Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1362618&r1=1362617&r2=1362618&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Tue Jul 17 19:32:14 2012 @@ -380,7 +380,7 @@ public class TestLogAggregationService e @Test @SuppressWarnings("unchecked") - public void testLogAggregationInitFailsWithoutKillingNM() throws Exception { + public void testLogAggregationFailsWithoutKillingNM() throws Exception { this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, @@ -412,7 +412,16 @@ public class TestLogAggregationService e new ApplicationFinishEvent(appId, "Application failed to init aggregation: KABOOM!") }; checkEvents(appEventHandler, expectedEvents, false, - "getType", "getApplicationID", "getDiagnostic"); + "getType", "getApplicationID", "getDiagnostic"); + + // verify trying to collect logs for containers/apps we don't know about + // doesn't blow up and tear down the NM + logAggregationService.handle(new LogHandlerContainerFinishedEvent( + BuilderUtils.newContainerId(4, 1, 1, 1), 0)); + dispatcher.await(); + logAggregationService.handle(new LogHandlerAppFinishedEvent( + BuilderUtils.newApplicationId(1, 5))); + dispatcher.await(); } private void writeContainerLogs(File appLogDir, ContainerId containerId)