Author: daryn
Date: Tue Jul 17 19:32:14 2012
New Revision: 1362618
URL: http://svn.apache.org/viewvc?rev=1362618&view=rev
Log:
MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't init.
(Jason Lowe via daryn)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1362618&r1=1362617&r2=1362618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 17
19:32:14 2012
@@ -724,6 +724,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4283. Display tail of aggregated logs by default (Jason Lowe via
bobby)
+ MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't
+ init. (Jason Lowe via daryn)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1362618&r1=1362617&r2=1362618&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
Tue Jul 17 19:32:14 2012
@@ -342,14 +342,14 @@ public class LogAggregationService exten
// A container is complete. Put this containers' logs up for aggregation if
// this containers' logs are needed.
- if (!this.appLogAggregators.containsKey(
- containerId.getApplicationAttemptId().getApplicationId())) {
- throw new YarnException("Application is not initialized yet for "
- + containerId);
+ AppLogAggregator aggregator = this.appLogAggregators.get(
+ containerId.getApplicationAttemptId().getApplicationId());
+ if (aggregator == null) {
+ LOG.warn("Log aggregation is not initialized for " + containerId
+ + ", did it fail to start?");
+ return;
}
- this.appLogAggregators.get(
- containerId.getApplicationAttemptId().getApplicationId())
- .startContainerLogAggregation(containerId, exitCode == 0);
+ aggregator.startContainerLogAggregation(containerId, exitCode == 0);
}
private void stopApp(ApplicationId appId) {
@@ -357,11 +357,13 @@ public class LogAggregationService exten
// App is complete. Finish up any containers' pending log aggregation and
// close the application specific logFile.
- if (!this.appLogAggregators.containsKey(appId)) {
- throw new YarnException("Application is not initialized yet for "
- + appId);
+ AppLogAggregator aggregator = this.appLogAggregators.get(appId);
+ if (aggregator == null) {
+ LOG.warn("Log aggregation is not initialized for " + appId
+ + ", did it fail to start?");
+ return;
}
- this.appLogAggregators.get(appId).finishLogAggregation();
+ aggregator.finishLogAggregation();
}
@Override
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1362618&r1=1362617&r2=1362618&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
Tue Jul 17 19:32:14 2012
@@ -380,7 +380,7 @@ public class TestLogAggregationService e
@Test
@SuppressWarnings("unchecked")
- public void testLogAggregationInitFailsWithoutKillingNM() throws Exception {
+ public void testLogAggregationFailsWithoutKillingNM() throws Exception {
this.conf.set(YarnConfiguration.NM_LOG_DIRS,
localLogDir.getAbsolutePath());
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
@@ -412,7 +412,16 @@ public class TestLogAggregationService e
new ApplicationFinishEvent(appId, "Application failed to init
aggregation: KABOOM!")
};
checkEvents(appEventHandler, expectedEvents, false,
- "getType", "getApplicationID", "getDiagnostic");
+ "getType", "getApplicationID", "getDiagnostic");
+
+ // verify trying to collect logs for containers/apps we don't know about
+ // doesn't blow up and tear down the NM
+ logAggregationService.handle(new LogHandlerContainerFinishedEvent(
+ BuilderUtils.newContainerId(4, 1, 1, 1), 0));
+ dispatcher.await();
+ logAggregationService.handle(new LogHandlerAppFinishedEvent(
+ BuilderUtils.newApplicationId(1, 5)));
+ dispatcher.await();
}
private void writeContainerLogs(File appLogDir, ContainerId containerId)
