Author: vinodkv
Date: Tue Jul 16 23:45:46 2013
New Revision: 1503949
URL: http://svn.apache.org/r1503949
Log:
YARN-820. Fixed an invalid state transition in NodeManager caused by failing
resource localization. Contributed by Mayank Bansal.
svn merge --ignore-ancestry -c 1503947 ../../trunk/
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1503949&r1=1503948&r2=1503949&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Tue
Jul 16 23:45:46 2013
@@ -24,6 +24,9 @@ Release 2.1.1-beta - UNRELEASED
YARN-661. Fixed NM to cleanup users' local directories correctly when
starting up. (Omkar Vinit Joshi via vinodkv)
+ YARN-820. Fixed an invalid state transition in NodeManager caused by
failing
+ resource localization. (Mayank Bansal via vinodkv)
+
Release 2.1.0-beta - 2013-07-02
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1503949&r1=1503948&r2=1503949&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
Tue Jul 16 23:45:46 2013
@@ -290,6 +290,11 @@ public class ContainerImpl implements Co
.addTransition(ContainerState.DONE, ContainerState.DONE,
ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
UPDATE_DIAGNOSTICS_TRANSITION)
+ // This transition may result when
+ // we notify container of failed localization if localizer thread (for
+ // that container) fails for some reason
+ .addTransition(ContainerState.DONE, ContainerState.DONE,
+ ContainerEventType.RESOURCE_FAILED)
// create the topology tables
.installTopology();
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java?rev=1503949&r1=1503948&r2=1503949&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
Tue Jul 16 23:45:46 2013
@@ -112,12 +112,17 @@ public class LocalizedResource implement
.append(getState() == ResourceState.LOCALIZED
? getLocalPath() + "," + getSize()
: "pending").append(",[");
- for (ContainerId c : ref) {
- sb.append("(").append(c.toString()).append(")");
+ try {
+ this.readLock.lock();
+ for (ContainerId c : ref) {
+ sb.append("(").append(c.toString()).append(")");
+ }
+ sb.append("],").append(getTimestamp()).append(",").append(getState())
+ .append("}");
+ return sb.toString();
+ } finally {
+ this.readLock.unlock();
}
- sb.append("],").append(getTimestamp()).append(",")
- .append(getState()).append("}");
- return sb.toString();
}
private void release(ContainerId container) {
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1503949&r1=1503948&r2=1503949&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
Tue Jul 16 23:45:46 2013
@@ -240,6 +240,32 @@ public class TestContainer {
}
}
}
+
+ @Test
+ @SuppressWarnings("unchecked")
+ // mocked generic
+ public void testLocalizationFailureAtDone() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(6, 314159265358979L, 4344, "yak");
+ wc.initContainer();
+ wc.localizeResources();
+ wc.launchContainer();
+ reset(wc.localizerBus);
+ wc.containerSuccessful();
+ wc.containerResourcesCleanup();
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ // Now in DONE, issue RESOURCE_FAILED as done by LocalizeRunner
+ wc.resourceFailedContainer();
+ // Verify still in DONE
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ verifyCleanupCall(wc);
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
@Test
@SuppressWarnings("unchecked") // mocked generic
@@ -624,6 +650,11 @@ public class TestContainer {
drainDispatcherEvents();
}
+ public void resourceFailedContainer() {
+ c.handle(new ContainerEvent(cId, ContainerEventType.RESOURCE_FAILED));
+ drainDispatcherEvents();
+ }
+
// Localize resources
// Skip some resources so as to consider them failed
public Map<Path, List<String>> doLocalizeResources(
