Author: tucu
Date: Tue Sep 10 12:13:19 2013
New Revision: 1521453
URL: http://svn.apache.org/r1521453
Log:
YARN-910. Augmented auxiliary services to listen for container starts and
completions in addition to application events. (Alejandro Abdelnur via vinodkv)
Added:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java
- copied unchanged from r1521298,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java
- copied unchanged from r1521298,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java
- copied unchanged from r1521298,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1521453&r1=1521452&r2=1521453&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Tue
Sep 10 12:13:19 2013
@@ -59,6 +59,10 @@ Release 2.1.1-beta - UNRELEASED
YARN-696. Changed RMWebservice apps call to take in multiple application
states. (Trevor Lorimer via vinodkv)
+ YARN-910. Augmented auxiliary services to listen for container starts and
+ completions in addition to application events. (Alejandro Abdelnur via
+ vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java?rev=1521453&r1=1521452&r2=1521453&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java
Tue Sep 10 12:13:19 2013
@@ -79,4 +79,27 @@ public abstract class AuxiliaryService e
* applications.
*/
public abstract ByteBuffer getMetaData();
-}
\ No newline at end of file
+
+ /**
+ * A new container is started on this NodeManager. This is a signal to
+ * this {@link AuxiliaryService} about the container initialization.
+ * This method is called when the NodeManager receives the container launch
+ * command from the ApplicationMaster and before the container process is
+ * launched.
+ *
+ * @param initContainerContext context for the container's initialization
+ */
+ public void initializeContainer(ContainerInitializationContext
+ initContainerContext) {
+ }
+
+ /**
+ * A container is finishing on this NodeManager. This is a signal to this
+ * {@link AuxiliaryService} about the same.
+ *
+ * @param stopContainerContext context for the container termination
+ */
+ public void stopContainer(ContainerTerminationContext stopContainerContext) {
+ }
+
+}
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java?rev=1521453&r1=1521452&r2=1521453&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
Tue Sep 10 12:13:19 2013
@@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.event.Even
import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
import org.apache.hadoop.yarn.server.api.AuxiliaryService;
import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
+import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
+import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
public class AuxServices extends AbstractService
implements ServiceStateChangeListener, EventHandler<AuxServicesEvent> {
@@ -178,7 +180,21 @@ public class AuxServices extends Abstrac
.getApplicationID()));
}
break;
- default:
+ case CONTAINER_INIT:
+ for (AuxiliaryService serv : serviceMap.values()) {
+ serv.initializeContainer(new ContainerInitializationContext(
+ event.getUser(), event.getContainer().getContainerId(),
+ event.getContainer().getResource()));
+ }
+ break;
+ case CONTAINER_STOP:
+ for (AuxiliaryService serv : serviceMap.values()) {
+ serv.stopContainer(new ContainerTerminationContext(
+ event.getUser(), event.getContainer().getContainerId(),
+ event.getContainer().getResource()));
+ }
+ break;
+ default:
throw new RuntimeException("Unknown type: " + event.getType());
}
}
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java?rev=1521453&r1=1521452&r2=1521453&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java
Tue Sep 10 12:13:19 2013
@@ -21,7 +21,10 @@ package org.apache.hadoop.yarn.server.no
import java.nio.ByteBuffer;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .Container;
public class AuxServicesEvent extends AbstractEvent<AuxServicesEventType> {
@@ -29,18 +32,30 @@ public class AuxServicesEvent extends Ab
private final String serviceId;
private final ByteBuffer serviceData;
private final ApplicationId appId;
+ private final Container container;
public AuxServicesEvent(AuxServicesEventType eventType, ApplicationId appId)
{
this(eventType, null, appId, null, null);
}
+ public AuxServicesEvent(AuxServicesEventType eventType, Container container)
{
+ this(eventType, null, container.getContainerId().getApplicationAttemptId()
+ .getApplicationId(), null, null, container);
+ }
+
public AuxServicesEvent(AuxServicesEventType eventType, String user,
ApplicationId appId, String serviceId, ByteBuffer serviceData) {
+ this(eventType, user, appId, serviceId, serviceData, null);
+ }
+ public AuxServicesEvent(AuxServicesEventType eventType, String user,
+ ApplicationId appId, String serviceId, ByteBuffer serviceData,
+ Container container) {
super(eventType);
this.user = user;
this.appId = appId;
this.serviceId = serviceId;
this.serviceData = serviceData;
+ this.container = container;
}
public String getServiceID() {
@@ -59,4 +74,8 @@ public class AuxServicesEvent extends Ab
return appId;
}
+ public Container getContainer() {
+ return container;
+ }
+
}
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java?rev=1521453&r1=1521452&r2=1521453&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java
Tue Sep 10 12:13:19 2013
@@ -20,5 +20,7 @@ package org.apache.hadoop.yarn.server.no
public enum AuxServicesEventType {
APPLICATION_INIT,
- APPLICATION_STOP
+ APPLICATION_STOP,
+ CONTAINER_INIT,
+ CONTAINER_STOP
}
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1521453&r1=1521452&r2=1521453&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
Tue Sep 10 12:13:19 2013
@@ -503,6 +503,9 @@ public class ContainerImpl implements Co
final ContainerLaunchContext ctxt = container.launchContext;
container.metrics.initingContainer();
+ container.dispatcher.getEventHandler().handle(new AuxServicesEvent
+ (AuxServicesEventType.CONTAINER_INIT, container));
+
// Inform the AuxServices about the opaque serviceData
Map<String,ByteBuffer> csd = ctxt.getServiceData();
if (csd != null) {
@@ -820,8 +823,16 @@ public class ContainerImpl implements Co
static class ContainerDoneTransition implements
SingleArcTransition<ContainerImpl, ContainerEvent> {
@Override
+ @SuppressWarnings("unchecked")
public void transition(ContainerImpl container, ContainerEvent event) {
container.finished();
+ //if the current state is NEW it means the CONTAINER_INIT was never
+ // sent for the event, thus no need to send the CONTAINER_STOP
+ if (container.getCurrentState()
+ != org.apache.hadoop.yarn.api.records.ContainerState.NEW) {
+ container.dispatcher.getEventHandler().handle(new AuxServicesEvent
+ (AuxServicesEventType.CONTAINER_STOP, container));
+ }
}
}
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java?rev=1521453&r1=1521452&r2=1521453&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
Tue Sep 10 12:13:19 2013
@@ -22,6 +22,7 @@ import static org.apache.hadoop.service.
import static org.apache.hadoop.service.Service.STATE.STARTED;
import static org.apache.hadoop.service.Service.STATE.STOPPED;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -34,11 +35,21 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
import org.apache.hadoop.yarn.server.api.AuxiliaryService;
+import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
+import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .ContainerImpl;
import org.junit.Test;
public class TestAuxServices {
@@ -52,8 +63,10 @@ public class TestAuxServices {
private int remaining_stop;
private ByteBuffer meta = null;
private ArrayList<Integer> stoppedApps;
+ private ContainerId containerId;
+ private Resource resource;
- LightService(String name, char idef, int expected_appId) {
+ LightService(String name, char idef, int expected_appId) {
this(name, idef, expected_appId, null);
}
LightService(String name, char idef, int expected_appId, ByteBuffer meta) {
@@ -95,7 +108,22 @@ public class TestAuxServices {
public ByteBuffer getMetaData() {
return meta;
}
- }
+
+ @Override
+ public void initializeContainer(
+ ContainerInitializationContext initContainerContext) {
+ containerId = initContainerContext.getContainerId();
+ resource = initContainerContext.getResource();
+ }
+
+ @Override
+ public void stopContainer(
+ ContainerTerminationContext stopContainerContext) {
+ containerId = stopContainerContext.getContainerId();
+ resource = stopContainerContext.getResource();
+ }
+
+ }
static class ServiceA extends LightService {
public ServiceA() {
@@ -142,6 +170,35 @@ public class TestAuxServices {
assertEquals("app not properly stopped", 1, appIds.size());
assertTrue("wrong app stopped", appIds.contains((Integer)66));
}
+
+ for (AuxiliaryService serv : servs) {
+ assertNull(((LightService) serv).containerId);
+ assertNull(((LightService) serv).resource);
+ }
+
+
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId1,
1);
+ ContainerTokenIdentifier cti = new ContainerTokenIdentifier(
+ ContainerId.newInstance(attemptId, 1), "", "",
+ Resource.newInstance(1, 1), 0,0,0);
+ Container container = new ContainerImpl(null, null, null, null, null, cti);
+ ContainerId containerId = container.getContainerId();
+ Resource resource = container.getResource();
+ event = new
AuxServicesEvent(AuxServicesEventType.CONTAINER_INIT,container);
+ aux.handle(event);
+ for (AuxiliaryService serv : servs) {
+ assertEquals(containerId, ((LightService) serv).containerId);
+ assertEquals(resource, ((LightService) serv).resource);
+ ((LightService) serv).containerId = null;
+ ((LightService) serv).resource = null;
+ }
+
+ event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_STOP,
container);
+ aux.handle(event);
+ for (AuxiliaryService serv : servs) {
+ assertEquals(containerId, ((LightService) serv).containerId);
+ assertEquals(resource, ((LightService) serv).resource);
+ }
}
@Test
