Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry b0025d60d -> a0528889f
SLIDER-348. Use command timeout specified in the metainfo Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d66d4444 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d66d4444 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d66d4444 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: d66d4444fd0a3da9aa0f34b354c530a323081ff2 Parents: 111e03f Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Thu Aug 21 23:30:17 2014 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Thu Aug 21 23:30:17 2014 -0700 ---------------------------------------------------------------------- .../providers/agent/AgentProviderService.java | 44 +++++++++++++------- .../agent/application/metadata/Component.java | 6 +++ .../agent/TestAgentProviderService.java | 32 ++++++++++---- 3 files changed, 59 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d66d4444/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index ec5425a..562ca1f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -56,6 +56,7 @@ import org.apache.slider.providers.ProviderCore; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.agent.application.metadata.Application; +import org.apache.slider.providers.agent.application.metadata.CommandScript; import org.apache.slider.providers.agent.application.metadata.Component; import org.apache.slider.providers.agent.application.metadata.ComponentExport; import org.apache.slider.providers.agent.application.metadata.ConfigFile; @@ -454,13 +455,20 @@ public class AgentProviderService extends AbstractProviderService implements String containerId = getContainerId(label); StateAccessForProviders accessor = getAmState(); - String scriptPath = getScriptPathFromMetainfo(roleName); + CommandScript cmdScript = getScriptPathFromMetainfo(roleName); - if (scriptPath == null) { + if (cmdScript == null || cmdScript.getScript() == null) { log.error("role.script is unavailable for " + roleName + ". Commands will not be sent."); return response; } + String scriptPath = cmdScript.getScript(); + long timeout = cmdScript.getTimeout(); + + if(timeout == 0L) { + timeout = 600L; + } + if (!getComponentStatuses().containsKey(label)) { return response; } @@ -502,14 +510,14 @@ public class AgentProviderService extends AbstractProviderService implements if (Command.NOP != command) { if (command == Command.INSTALL) { log.info("Installing {} on {}.", roleName, containerId); - addInstallCommand(roleName, containerId, response, scriptPath); + addInstallCommand(roleName, containerId, response, scriptPath, timeout); componentStatus.commandIssued(command); } else if (command == Command.START) { // check against dependencies boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values()); if (canExecute) { log.info("Starting {} on {}.", roleName, containerId); - addStartCommand(roleName, containerId, response, scriptPath, isMarkedAutoRestart(roleName)); + addStartCommand(roleName, containerId, response, scriptPath, timeout, isMarkedAutoRestart(roleName)); componentStatus.commandIssued(command); } else { log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId); @@ -1037,10 +1045,10 @@ public class AgentProviderService extends AbstractProviderService implements * * @return */ - protected String getScriptPathFromMetainfo(String roleName) { + protected CommandScript getScriptPathFromMetainfo(String roleName) { Component component = getApplicationComponent(roleName); if (component != null) { - return component.getCommandScript().getScript(); + return component.getCommandScript(); } return null; } @@ -1134,7 +1142,11 @@ public class AgentProviderService extends AbstractProviderService implements * @throws SliderException */ @VisibleForTesting - protected void addInstallCommand(String roleName, String containerId, HeartBeatResponse response, String scriptPath) + protected void addInstallCommand(String roleName, + String containerId, + HeartBeatResponse response, + String scriptPath, + long timeout) throws SliderException { assert getAmState().isApplicationLive(); ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); @@ -1155,7 +1167,7 @@ public class AgentProviderService extends AbstractProviderService implements setInstallCommandConfigurations(cmd, containerId); - cmd.setCommandParams(setCommandParameters(scriptPath, false)); + cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false)); cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME)); response.addExecutionCommand(cmd); @@ -1191,13 +1203,13 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setCommandId(cmd.getTaskId() + "-1"); } - private Map<String, String> setCommandParameters(String scriptPath, boolean recordConfig) { + private Map<String, String> setCommandParameters(String scriptPath, long timeout, boolean recordConfig) { Map<String, String> cmdParams = new TreeMap<String, String>(); cmdParams.put("service_package_folder", "${AGENT_WORK_ROOT}/work/app/definition/package"); cmdParams.put("script", scriptPath); cmdParams.put("schema_version", "2.0"); - cmdParams.put("command_timeout", "300"); + cmdParams.put("command_timeout", Long.toString(timeout)); cmdParams.put("script_type", "PYTHON"); cmdParams.put("record_config", Boolean.toString(recordConfig)); return cmdParams; @@ -1210,7 +1222,11 @@ public class AgentProviderService extends AbstractProviderService implements } @VisibleForTesting - protected void addStatusCommand(String roleName, String containerId, HeartBeatResponse response, String scriptPath) + protected void addStatusCommand(String roleName, + String containerId, + HeartBeatResponse response, + String scriptPath, + long timeout) throws SliderException { assert getAmState().isApplicationLive(); ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); @@ -1229,7 +1245,7 @@ public class AgentProviderService extends AbstractProviderService implements hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); - cmd.setCommandParams(setCommandParameters(scriptPath, false)); + cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false)); Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId); @@ -1262,7 +1278,7 @@ public class AgentProviderService extends AbstractProviderService implements @VisibleForTesting protected void addStartCommand(String roleName, String containerId, HeartBeatResponse response, - String scriptPath, boolean isMarkedAutoRestart) + String scriptPath, long timeout, boolean isMarkedAutoRestart) throws SliderException { assert getAmState().isApplicationLive(); @@ -1288,7 +1304,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setRoleParams(roleParams); cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart)); - cmd.setCommandParams(setCommandParameters(scriptPath, true)); + cmd.setCommandParams(setCommandParameters(scriptPath, timeout, true)); Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d66d4444/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java index 03c64d4..1117ff1 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java @@ -124,4 +124,10 @@ public class Component { sb.append('}'); return sb.toString(); } + + class AutoRestartSettings { + private boolean requiresAutoRestart; + private int maxFailures; + private int inThisManyMinutes; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d66d4444/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index e0a9214..b0b059d 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -42,6 +42,7 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.providers.agent.application.metadata.Application; import org.apache.slider.providers.agent.application.metadata.CommandOrder; +import org.apache.slider.providers.agent.application.metadata.CommandScript; import org.apache.slider.providers.agent.application.metadata.Component; import org.apache.slider.providers.agent.application.metadata.ComponentExport; import org.apache.slider.providers.agent.application.metadata.ConfigFile; @@ -83,6 +84,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.expect; @@ -278,7 +280,9 @@ public class TestAgentProviderService { AgentProviderService mockAps = Mockito.spy(aps); doReturn(access).when(mockAps).getAmState(); - doReturn("scripts/hbase_master.py").when(mockAps).getScriptPathFromMetainfo(anyString()); + CommandScript cs = new CommandScript(); + cs.setScript("scripts/hbase_master.py"); + doReturn(cs).when(mockAps).getScriptPathFromMetainfo(anyString()); Metainfo metainfo = new Metainfo(); metainfo.setApplication(new Application()); doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString()); @@ -293,7 +297,8 @@ public class TestAgentProviderService { eq("HBASE_MASTER"), eq("mockcontainer_1"), any(HeartBeatResponse.class), - eq("scripts/hbase_master.py")); + eq("scripts/hbase_master.py"), + eq(600L)); doReturn(conf).when(mockAps).getConfig(); } catch (SliderException e) { } @@ -612,8 +617,8 @@ public class TestAgentProviderService { AgentProviderService aps = new AgentProviderService(); AgentProviderService mockAps = Mockito.spy(aps); doReturn(metainfo).when(mockAps).getMetainfo(); - String scriptPath = mockAps.getScriptPathFromMetainfo("HBASE_MASTER"); - Assert.assertEquals(scriptPath, "scripts/hbase_master.py"); + CommandScript script = mockAps.getScriptPathFromMetainfo("HBASE_MASTER"); + Assert.assertEquals(script.getScript(), "scripts/hbase_master.py"); String metainfo_1_str_bad = "<metainfo>\n" + " <schemaVersion>2.0</schemaVersion>\n" @@ -712,12 +717,14 @@ public class TestAgentProviderService { anyString(), anyString(), any(HeartBeatResponse.class), - anyString()); + anyString(), + Mockito.anyLong()); doNothing().when(mockAps).addStartCommand( anyString(), anyString(), any(HeartBeatResponse.class), anyString(), + Mockito.anyLong(), Matchers.anyBoolean()); doNothing().when(mockAps).addGetConfigCommand( anyString(), @@ -792,7 +799,8 @@ public class TestAgentProviderService { Mockito.verify(mockAps, Mockito.times(1)).addInstallCommand(anyString(), anyString(), any(HeartBeatResponse.class), - anyString()); + anyString(), + Mockito.anyLong()); hb = new HeartBeat(); hb.setResponseId(1); @@ -802,7 +810,8 @@ public class TestAgentProviderService { Mockito.verify(mockAps, Mockito.times(2)).addInstallCommand(anyString(), anyString(), any(HeartBeatResponse.class), - anyString()); + anyString(), + Mockito.anyLong()); // RS succeeds install but does not start hb = new HeartBeat(); hb.setResponseId(2); @@ -821,6 +830,7 @@ public class TestAgentProviderService { anyString(), any(HeartBeatResponse.class), anyString(), + Mockito.anyLong(), Matchers.anyBoolean()); // RS still does not start hb = new HeartBeat(); @@ -832,6 +842,7 @@ public class TestAgentProviderService { anyString(), any(HeartBeatResponse.class), anyString(), + Mockito.anyLong(), Matchers.anyBoolean()); // MASTER succeeds install and issues start @@ -852,6 +863,7 @@ public class TestAgentProviderService { anyString(), any(HeartBeatResponse.class), anyString(), + Mockito.anyLong(), Matchers.anyBoolean()); Map<String, String> allocatedPorts = mockAps.getAllocatedPorts(); Assert.assertTrue(allocatedPorts != null); @@ -868,6 +880,7 @@ public class TestAgentProviderService { anyString(), any(HeartBeatResponse.class), anyString(), + Mockito.anyLong(), Matchers.anyBoolean()); // MASTER succeeds start hb = new HeartBeat(); @@ -893,6 +906,7 @@ public class TestAgentProviderService { anyString(), any(HeartBeatResponse.class), anyString(), + Mockito.anyLong(), Matchers.anyBoolean()); // JDK7 } catch (SliderException he) { @@ -987,7 +1001,7 @@ public class TestAgentProviderService { replay(access); - mockAps.addInstallCommand("HBASE_MASTER", "cid1", hbr, ""); + mockAps.addInstallCommand("HBASE_MASTER", "cid1", hbr, "", 0); ExecutionCommand cmd = hbr.getExecutionCommands().get(0); String pkgs = cmd.getHostLevelParams().get(AgentKeys.PACKAGE_LIST); Assert.assertEquals("[{\"type\":\"tarball\",\"name\":\"files/hbase-0.96.1-hadoop2-bin.tar.gz\"}]", pkgs); @@ -1062,7 +1076,7 @@ public class TestAgentProviderService { replay(access); - mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", Boolean.FALSE); + mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", 0, Boolean.FALSE); Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("hbase-site")); Map<String, String> hbaseSiteConf = hbr.getExecutionCommands().get(0).getConfigurations().get("hbase-site"); Assert.assertTrue(hbaseSiteConf.containsKey("a.port"));
