SLIDER-280. Add unit and functional tests for agents discovering AM after restart (Gour Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a509561f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a509561f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a509561f Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: a509561f0c710e61f1f7612e14bc6c70176d29f9 Parents: fa7130d Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Wed Aug 20 15:07:21 2014 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Thu Sep 11 10:40:00 2014 -0700 ---------------------------------------------------------------------- .../web/rest/agent/ExecutionCommand.java | 16 ++ .../appmaster/web/rest/agent/StatusCommand.java | 13 ++ .../agent/TestAgentProviderService.java | 214 +++++++++++++++++++ 3 files changed, 243 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a509561f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java index c184e63..94aa29f 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java @@ -179,4 +179,20 @@ public class ExecutionCommand { public void setComponentName(String componentName) { this.componentName = componentName; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("ExecutionCommand [commandType=").append(commandType) + .append(", clusterName=").append(clusterName).append(", taskId=") + .append(taskId).append(", commandId=").append(commandId) + .append(", hostname=").append(hostname).append(", role=").append(role) + .append(", hostLevelParams=").append(hostLevelParams) + .append(", roleParams=").append(roleParams).append(", roleCommand=") + .append(roleCommand).append(", configurations=").append(configurations) + .append(", commandParams=").append(commandParams) + .append(", serviceName=").append(serviceName) + .append(", componentName=").append(componentName).append("]"); + return builder.toString(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a509561f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java index 0610cac..52fb8dd 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java @@ -125,4 +125,17 @@ public class StatusCommand { public void setRoleCommand(String roleCommand) { this.roleCommand = roleCommand; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("StatusCommand [agentCommandType=").append(agentCommandType) + .append(", clusterName=").append(clusterName).append(", serviceName=") + .append(serviceName).append(", componentName=").append(componentName) + .append(", configurations=").append(configurations) + .append(", commandParams=").append(commandParams) + .append(", hostLevelParams=").append(hostLevelParams) + .append(", roleCommand=").append(roleCommand).append("]"); + return builder.toString(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a509561f/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index 3711e12..48a5f18 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterDescriptionKeys; import org.apache.slider.api.ClusterNode; @@ -41,6 +42,7 @@ import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.agent.application.metadata.Application; import org.apache.slider.providers.agent.application.metadata.CommandOrder; import org.apache.slider.providers.agent.application.metadata.CommandScript; @@ -58,6 +60,7 @@ import org.apache.slider.server.appmaster.model.mock.MockFileSystem; import org.apache.slider.server.appmaster.model.mock.MockNodeId; import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType; import org.apache.slider.server.appmaster.web.rest.agent.CommandReport; import org.apache.slider.server.appmaster.web.rest.agent.ComponentStatus; import org.apache.slider.server.appmaster.web.rest.agent.ExecutionCommand; @@ -381,6 +384,217 @@ public class TestAgentProviderService { Assert.assertEquals(2, hbr.getResponseId()); } + private AggregateConf prepareConfForAgentStateTests() { + ConfTree tree = new ConfTree(); + tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); + + AggregateConf instanceDefinition = new AggregateConf(); + instanceDefinition.setInternal(tree); + instanceDefinition.setAppConf(tree); + instanceDefinition.getAppConfOperations().getGlobalOptions() + .put(AgentKeys.APP_DEF, "."); + instanceDefinition.getAppConfOperations().getGlobalOptions() + .put(AgentKeys.AGENT_CONF, "."); + instanceDefinition.getAppConfOperations().getGlobalOptions() + .put(AgentKeys.AGENT_VERSION, "."); + return instanceDefinition; + } + + private AgentProviderService prepareProviderServiceForAgentStateTests() + throws IOException { + ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class); + Container container = createNiceMock(Container.class); + String role = "HBASE_MASTER"; + SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class); + FileSystem mockFs = new MockFileSystem(); + expect(sliderFileSystem.getFileSystem()).andReturn( + new FilterFileSystem(mockFs)).anyTimes(); + expect( + sliderFileSystem.createAmResource(anyObject(Path.class), + anyObject(LocalResourceType.class))).andReturn( + createNiceMock(LocalResource.class)).anyTimes(); + expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes(); + expect(container.getNodeId()).andReturn(new MockNodeId("localhost")) + .anyTimes(); + expect(container.getPriority()).andReturn(Priority.newInstance(1)); + + StateAccessForProviders access = createNiceMock(StateAccessForProviders.class); + AgentProviderService aps = new AgentProviderService(); + AgentProviderService mockAps = Mockito.spy(aps); + + doReturn(access).when(mockAps).getAmState(); + doReturn("scripts/hbase_master.py").when(mockAps) + .getScriptPathFromMetainfo(anyString()); + Metainfo metainfo = new Metainfo(); + Application application = new Application(); + metainfo.setApplication(application); + doReturn(metainfo).when(mockAps).getApplicationMetainfo( + any(SliderFileSystem.class), anyString()); + doReturn(metainfo).when(mockAps).getMetainfo(); + + Configuration conf = new Configuration(); + conf.set(SliderXmlConfKeys.REGISTRY_PATH, + SliderXmlConfKeys.DEFAULT_REGISTRY_PATH); + + try { + doReturn(true).when(mockAps).isMaster(anyString()); + doNothing().when(mockAps).addInstallCommand(eq("HBASE_MASTER"), + eq("mockcontainer_1"), any(HeartBeatResponse.class), + eq("scripts/hbase_master.py")); + doReturn(conf).when(mockAps).getConfig(); + } catch (SliderException e) { + } + + doNothing().when(mockAps).processAllocatedPorts(anyString(), anyString(), + anyString(), anyMap()); + expect(access.isApplicationLive()).andReturn(true).anyTimes(); + ClusterDescription desc = new ClusterDescription(); + desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181"); + desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE"); + expect(access.getClusterStatus()).andReturn(desc).anyTimes(); + + AggregateConf aggConf = new AggregateConf(); + ConfTreeOperations treeOps = aggConf.getAppConfOperations(); + treeOps.getOrAddComponent("HBASE_MASTER") + .put(AgentKeys.WAIT_HEARTBEAT, "0"); + treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE"); + treeOps.set("java_home", "/usr/jdk7/"); + treeOps.set("site.fs.defaultFS", "hdfs://c6409.ambari.apache.org:8020"); + treeOps.set(InternalKeys.INTERNAL_DATA_DIR_PATH, "hdfs://c6409.ambari.apache.org:8020/user/yarn/.slider/cluster/cl1/data"); + expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf); + expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes(); + expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes(); + replay(access, ctx, container, sliderFileSystem); + + List<Container> containers = new ArrayList<Container>(); + containers.add(container); + Map<Integer, ProviderRole> providerRoleMap = new HashMap<Integer, ProviderRole>(); + ProviderRole providerRole = new ProviderRole(role, 1); + providerRoleMap.put(1, providerRole); + mockAps.rebuildContainerDetails(containers, "mockcontainer_1", + providerRoleMap); + return mockAps; + } + + @Test + public void testAgentStateStarted() throws IOException, SliderException { + AggregateConf instanceDefinition = prepareConfForAgentStateTests(); + AgentProviderService mockAps = prepareProviderServiceForAgentStateTests(); + Register reg = new Register(); + reg.setResponseId(0); + reg.setHostname("mockcontainer_1___HBASE_MASTER"); + Map<String,String> ports = new HashMap<String,String>(); + ports.put("a","100"); + reg.setAllocatedPorts(ports); + + // Simulating agent in STARTED state + reg.setActualState(State.STARTED); + + mockAps.initializeApplicationConfiguration(instanceDefinition, + null); + + RegistrationResponse resp = mockAps.handleRegistration(reg); + Assert.assertEquals(0, resp.getResponseId()); + Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus()); + + Mockito.verify(mockAps, Mockito.times(1)).processAllocatedPorts( + anyString(), + anyString(), + anyString(), + anyMap() + ); + + HeartBeat hb = new HeartBeat(); + hb.setResponseId(1); + hb.setHostname("mockcontainer_1___HBASE_MASTER"); + HeartBeatResponse hbr = mockAps.handleHeartBeat(hb); + Assert.assertEquals(2, hbr.getResponseId()); + Assert.assertNotNull( + "Status command from AM cannot be null when agent's actualState " + + "is set to STARTED during registration", hbr.getStatusCommands()); + Assert.assertTrue( + "Status command from AM cannot be empty when agent's actualState " + + "is set to STARTED during registration", hbr.getStatusCommands() + .size() > 0); + Assert.assertEquals( + "AM should directly send a STATUS request if agent's actualState is " + + "set to STARTED during registration", + AgentCommandType.STATUS_COMMAND, hbr.getStatusCommands().get(0) + .getCommandType()); + Assert.assertEquals( + "AM should directly request for CONFIG if agent's actualState is " + + "set to STARTED during registration", + "GET_CONFIG", hbr.getStatusCommands().get(0) + .getRoleCommand()); + Assert.assertFalse("AM cannot ask agent to restart", hbr.isRestartAgent()); + } + + @Test + public void testAgentStateInstalled() throws IOException, SliderException { + AggregateConf instanceDefinition = prepareConfForAgentStateTests(); + AgentProviderService mockAps = prepareProviderServiceForAgentStateTests(); + + Metainfo metainfo = new Metainfo(); + Application application = new Application(); + CommandOrder cmdOrder = new CommandOrder(); + cmdOrder.setCommand("HBASE_MASTER-START"); + cmdOrder.setRequires("HBASE_MASTER-INSTALLED"); + application.addCommandOrder(cmdOrder); + metainfo.setApplication(application); + doReturn(metainfo).when(mockAps).getApplicationMetainfo( + any(SliderFileSystem.class), anyString()); + doReturn(metainfo).when(mockAps).getMetainfo(); + doNothing().when(mockAps).addRoleRelatedTokens(anyMap()); + + Register reg = new Register(); + reg.setResponseId(0); + reg.setHostname("mockcontainer_1___HBASE_MASTER"); + Map<String,String> ports = new HashMap<String,String>(); + ports.put("a","100"); + reg.setAllocatedPorts(ports); + + // Simulating agent in INSTALLED state + reg.setActualState(State.INSTALLED); + + mockAps.initializeApplicationConfiguration(instanceDefinition, + null); + + RegistrationResponse resp = mockAps.handleRegistration(reg); + Assert.assertEquals(0, resp.getResponseId()); + Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus()); + + Mockito.verify(mockAps, Mockito.times(1)).processAllocatedPorts( + anyString(), + anyString(), + anyString(), + anyMap() + ); + + HeartBeat hb = new HeartBeat(); + hb.setResponseId(1); + hb.setHostname("mockcontainer_1___HBASE_MASTER"); + HeartBeatResponse hbr = mockAps.handleHeartBeat(hb); + Assert.assertEquals(2, hbr.getResponseId()); + Assert.assertNotNull( + "Execution command from AM cannot be null when agent's actualState " + + "is set to INSTALLED during registration", hbr.getExecutionCommands()); + Assert.assertTrue( + "Execution command from AM cannot be empty when agent's actualState " + + "is set to INSTALLED during registration", hbr.getExecutionCommands() + .size() > 0); + Assert.assertEquals( + "AM should send an EXECUTION command if agent's actualState is " + + "set to INSTALLED during registration", + AgentCommandType.EXECUTION_COMMAND, hbr.getExecutionCommands().get(0) + .getCommandType()); + Assert.assertEquals( + "AM should request for START if agent's actualState is " + + "set to INSTALLED during registration", + "START", hbr.getExecutionCommands().get(0) + .getRoleCommand()); + Assert.assertFalse("AM cannot ask agent to restart", hbr.isRestartAgent()); + } + @Test public void testRoleHostMapping() throws Exception { AgentProviderService aps = new AgentProviderService();
