Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts: slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java slider-core/src/main/java/org/apache/slider/providers/ProviderService.java slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/434a964b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/434a964b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/434a964b Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 434a964b65fad17e8155b5f76f579ff675d32394 Parents: 5de46b8 7b7d29e Author: Steve Loughran <ste...@apache.org> Authored: Mon Aug 25 14:37:26 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Aug 25 14:37:26 2014 +0100 ---------------------------------------------------------------------- app-packages/accumulo/pom.xml | 2 +- .../command-logger/application-pkg/pom.xml | 2 +- .../command-logger/slider-pkg/appConfig.json | 2 - .../command-logger/slider-pkg/metainfo.xml | 8 + app-packages/command-logger/slider-pkg/pom.xml | 2 +- app-packages/hbase/appConfig.json | 38 +---- app-packages/hbase/configuration/global.xml | 160 ------------------- app-packages/hbase/configuration/hbase-env.xml | 117 ++++++++++++++ .../hbase/configuration/hbase-log4j.xml | 1 + app-packages/hbase/configuration/hbase-site.xml | 19 ++- app-packages/hbase/get-hbase-site.sh | 15 ++ app-packages/hbase/metainfo.xml | 24 +++ app-packages/hbase/package/scripts/hbase.py | 26 +-- app-packages/hbase/package/scripts/params.py | 26 ++- .../hbase/package/templates/hbase-env.sh.j2 | 81 ---------- app-packages/storm/appConfig.json | 88 +--------- app-packages/storm/configuration/global.xml | 39 ----- app-packages/storm/configuration/storm-env.xml | 40 +++++ app-packages/storm/configuration/storm-site.xml | 15 +- app-packages/storm/metainfo.xml | 14 ++ app-packages/storm/package/scripts/params.py | 8 +- app-packages/storm/package/scripts/storm.py | 7 +- .../storm/package/scripts/yaml_config.py | 7 +- pom.xml | 4 +- slider-agent/pom.xml | 2 +- .../src/main/python/agent/Controller.py | 2 +- slider-agent/src/main/python/agent/Registry.py | 12 +- slider-agent/src/main/python/agent/main.py | 7 +- .../core/providers/__init__.py | 19 ++- .../core/providers/system.py | 8 + .../core/resources/system.py | 1 + .../python/resource_management/TestPackage.py | 32 ++++ slider-assembly/pom.xml | 2 +- slider-assembly/src/conf/slider-client.xml | 32 ++-- slider-assembly/src/main/scripts/slider | 3 +- slider-assembly/src/main/scripts/slider.py | 21 ++- slider-core/pom.xml | 2 +- .../java/org/apache/slider/api/StatusKeys.java | 7 +- .../registry/info/CustomRegistryConstants.java | 9 +- .../providers/AbstractProviderService.java | 15 +- .../slider/providers/ProviderService.java | 15 +- .../providers/agent/AgentProviderService.java | 160 ++++++++++++++----- .../slider/providers/agent/AgentUtils.java | 31 +++- .../agent/application/metadata/Application.java | 11 +- .../agent/application/metadata/Component.java | 6 + .../agent/application/metadata/ConfigFile.java | 53 ++++++ .../metadata/ConfigurationDependencies.java | 39 ----- .../application/metadata/DefaultConfig.java | 39 +++++ .../metadata/DefaultConfigParser.java | 54 +++++++ .../application/metadata/MetainfoParser.java | 9 +- .../application/metadata/PropertyInfo.java | 54 +++++++ .../slideram/SliderAMProviderService.java | 28 ++-- .../server/appmaster/SliderAppMaster.java | 49 +++--- .../server/appmaster/web/AgentService.java | 28 +--- .../appmaster/web/rest/agent/AgentWebApp.java | 16 +- .../server/services/security/SecurityUtils.java | 60 ++++--- .../LaunchedWorkflowCompositeService.java | 4 + .../services/workflow/ClosingService.java | 3 + .../test_command_log/appConfig.json | 2 - .../test_command_log/appConfig_fast_no_reg.json | 2 - .../test_command_log/appConfig_no_hb.json | 2 - .../agent/AgentMiniClusterTestBase.groovy | 9 +- .../common/tools/TestWindowsSupport.groovy | 112 +++++++++++++ .../agent/TestAgentAMManagementWS.groovy | 2 +- .../model/mock/MockProviderService.groovy | 7 +- .../publisher/TestPublisherRestResources.groovy | 11 +- .../apache/slider/test/SliderTestUtils.groovy | 39 +++++ .../slider/test/YarnMiniClusterTestBase.groovy | 23 +-- .../agent/TestAgentProviderService.java | 101 ++++++++++-- .../application/metadata/TestConfigParser.java | 107 +++++++++++++ .../metadata/TestMetainfoParser.java | 7 + .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../workflow/TestWorkflowClosingService.java | 4 +- .../slider/common/tools/test/metainfo.xml | 11 +- slider-funtest/pom.xml | 2 +- .../accumulo/accumulo-funtests/pom.xml | 2 +- .../accumulo/slider-accumulo-provider/pom.xml | 2 +- .../providers/accumulo/AccumuloTestBase.groovy | 7 +- slider-providers/hbase/hbase-funtests/pom.xml | 2 +- .../hbase/slider-hbase-provider/pom.xml | 2 +- .../providers/hbase/HBaseProviderService.java | 27 ---- .../minicluster/HBaseMiniClusterTestBase.groovy | 9 +- 82 files changed, 1296 insertions(+), 777 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java ---------------------------------------------------------------------- diff --cc slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java index 89e5c65,baddb56..0e7ea25 --- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@@ -335,15 -331,13 +335,16 @@@ public abstract class AbstractProviderS } } } + @Override - public void applyInitialRegistryDefinitions(URL unsecureWebAPI, - URL secureWebAPI, - ServiceInstanceData registryInstanceData, - ServiceRecord serviceRecord) - throws IOException { - - this.amWebAPI = unsecureWebAPI; + public void applyInitialRegistryDefinitions(URL amWebURI, + URL agentOpsURI, + URL agentStatusURI, - ServiceInstanceData registryInstanceData) throws IOException { ++ ServiceInstanceData registryInstanceData, ++ ServiceRecord serviceRecord) ++ throws IOException { + + this.amWebAPI = amWebURI; this.registryInstanceData = registryInstanceData; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java ---------------------------------------------------------------------- diff --cc slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index 67ca0a9,a9aebad..b28ac6a --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@@ -184,15 -176,16 +184,18 @@@ public interface ProviderService extend /** * Prior to going live -register the initial service registry data - * @param unsecureWebAPI - * @param secureWebAPI + * @param amWebURI + * @param agentOpsURI + * @param agentStatusURI * @param registryInstanceData + * @param serviceRecord */ - void applyInitialRegistryDefinitions(URL unsecureWebAPI, - URL secureWebAPI, - ServiceInstanceData registryInstanceData, - ServiceRecord serviceRecord) throws IOException; + void applyInitialRegistryDefinitions(URL amWebURI, + URL agentOpsURI, + URL agentStatusURI, - ServiceInstanceData registryInstanceData) ++ ServiceInstanceData registryInstanceData, ++ ServiceRecord serviceRecord) + throws IOException; /** * Create the container release selector for this provider...any policy http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --cc slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 790e379,e79fb60..f78d468 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@@ -585,29 -600,25 +603,40 @@@ public class AgentProviderService exten } @Override - public void applyInitialRegistryDefinitions(URL unsecureWebAPI, - URL secureWebAPI, - ServiceInstanceData instanceData, ServiceRecord serviceRecord) throws - IOException { - super.applyInitialRegistryDefinitions(unsecureWebAPI, - secureWebAPI, - instanceData, - serviceRecord); - public void applyInitialRegistryDefinitions(URL amWebURI, - URL agentOpsURI, - URL agentStatusURI, - ServiceInstanceData instanceData) throws IOException { ++ public void applyInitialRegistryDefinitions (URL amWebURI, ++ URL agentOpsURI, ++ URL agentStatusURI, ++ ServiceInstanceData instanceData, ++ ServiceRecord serviceRecord) ++ throws IOException { + super.applyInitialRegistryDefinitions(amWebURI, + agentOpsURI, - agentStatusURI, instanceData); ++ agentStatusURI, ++ instanceData, ++ serviceRecord); try { - URL url = new URL(secureWebAPI, SLIDER_PATH_AGENTS); ++ URL restURL = new URL(agentOpsURI, SLIDER_PATH_AGENTS); + - URI uri = url.toURI(); instanceData.internalView.endpoints.put( - CustomRegistryConstants.AGENT_REST_API, + CustomRegistryConstants.AGENT_SECURE_REST_API, - new RegisteredEndpoint( - new URL(agentOpsURI, SLIDER_PATH_AGENTS), ++ new RegisteredEndpoint(restURL, + "Agent Secure REST API")); ++ URL agentStatusURL = new URL(agentStatusURI, SLIDER_PATH_AGENTS); + instanceData.internalView.endpoints.put( + CustomRegistryConstants.AGENT_ONEWAY_REST_API, new RegisteredEndpoint( - url, - "Agent REST API")); - new URL(agentStatusURI, SLIDER_PATH_AGENTS), ++ agentStatusURL, + "Agent Oneway REST API")); + + serviceRecord.addInternalEndpoint( - new Endpoint(CustomRegistryConstants.AGENT_REST_API, ++ new Endpoint(CustomRegistryConstants.AGENT_SECURE_REST_API, ++ ProtocolTypes.PROTOCOL_REST, ++ restURL.toURI())); ++ serviceRecord.addInternalEndpoint( ++ new Endpoint(CustomRegistryConstants.AGENT_ONEWAY_REST_API, + ProtocolTypes.PROTOCOL_REST, - uri)); ++ agentStatusURL.toURI())); } catch (URISyntaxException e) { throw new IOException(e); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java ---------------------------------------------------------------------- diff --cc slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java index ab41549,863ea7e..2b62d24 --- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java @@@ -112,15 -110,15 +112,17 @@@ public class SliderAMProviderService ex } @Override - public void applyInitialRegistryDefinitions(URL unsecureWebAPI, - URL secureWebAPI, + public void applyInitialRegistryDefinitions(URL amWebURI, - URL agentOpsURI, - URL agentStatusURI, - ServiceInstanceData instanceData) throws IOException { ++ URL agentOpsURI, ++ URL agentStatusURI, + ServiceInstanceData instanceData, - ServiceRecord serviceRecord) throws IOException { - super.applyInitialRegistryDefinitions(unsecureWebAPI, - secureWebAPI, - instanceData, ++ ServiceRecord serviceRecord) ++ throws IOException { + super.applyInitialRegistryDefinitions(amWebURI, - agentOpsURI, - agentStatusURI, instanceData - ); - ++ agentOpsURI, ++ agentStatusURI, ++ instanceData, + serviceRecord); - // now publish site.xml files YarnConfiguration defaultYarnConfig = new YarnConfiguration(); amState.getPublishedSliderConfigurations().put( @@@ -151,15 -149,10 +153,14 @@@ try { - RegistryView externalView = instanceData.externalView; RegisteredEndpoint webUI = - new RegisteredEndpoint(unsecureWebAPI, "Application Master Web UI"); + new RegisteredEndpoint(amWebURI, "Application Master Web UI"); - URL managementAPI = new URL(unsecureWebAPI, SLIDER_PATH_MANAGEMENT); - URL registryREST = new URL(unsecureWebAPI, RestPaths.SLIDER_PATH_REGISTRY + "/" + ++ URL managementAPI = new URL(amWebURI, SLIDER_PATH_MANAGEMENT); ++ URL registryREST = new URL(amWebURI, RestPaths.SLIDER_PATH_REGISTRY + "/" + + RestPaths.REGISTRY_SERVICE); - URL publisherURL = new URL(unsecureWebAPI, SLIDER_PATH_PUBLISHER); + + RegistryView externalView = instanceData.externalView; externalView.endpoints.put(CommonRegistryConstants.WEB_UI, webUI); externalView.endpoints.put( @@@ -171,9 -164,11 +172,10 @@@ externalView.endpoints.put( CustomRegistryConstants.REGISTRY_REST_API, new RegisteredEndpoint( - new URL(amWebURI, RestPaths.SLIDER_PATH_REGISTRY + "/" + - RestPaths.REGISTRY_SERVICE), + registryREST, "Registry Web Service" ) ); + URL publisherURL = new URL(amWebURI, SLIDER_PATH_PUBLISHER); externalView.endpoints.put( CustomRegistryConstants.PUBLISHER_REST_API, new RegisteredEndpoint( @@@ -185,20 -181,6 +187,18 @@@ externalView.configurationsURL = SliderUtils.appendToURL( publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET); + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.webEndpoint( - CommonRegistryConstants.WEB_UI, unsecureWebAPI.toURI())); ++ CommonRegistryConstants.WEB_UI, amWebURI.toURI())); + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.restEndpoint( + CustomRegistryConstants.MANAGEMENT_REST_API, + managementAPI.toURI())); + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.restEndpoint( + CustomRegistryConstants.PUBLISHER_REST_API, + publisherURL.toURI())); - - + } catch (URISyntaxException e) { throw new IOException(e); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index a387d3a,b4515f1..c743812 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@@ -871,11 -857,12 +876,12 @@@ public class SliderAppMaster extends Ab private void registerServiceInstance(String instanceName, ApplicationId appid) throws Exception { // the registry is running, so register services - URL unsecureWebAPI = new URL(appMasterTrackingUrl); - URL secureWebAPI = new URL(agentAccessUrl); + URL amWebURI = new URL(appMasterTrackingUrl); + URL agentOpsURI = new URL(agentOpsUrl); + URL agentStatusURI = new URL(agentStatusUrl); String serviceName = SliderKeys.APP_TYPE; int id = appid.getId(); - String appServiceType = RegistryNaming.createRegistryServiceType( + String serviceType = RegistryNaming.createRegistryServiceType( instanceName, service_user_name, serviceName); @@@ -907,52 -883,28 +913,55 @@@ CustomRegistryConstants.AM_IPC_PROTOCOL, new RegisteredEndpoint(rpcServiceAddress, RegisteredEndpoint.PROTOCOL_HADOOP_PROTOBUF, - "Slider AM RPC") ); + SLIDER_AM_RPC) ); + // Yarn registry + ServiceRecord serviceRecord = new ServiceRecord(); + serviceRecord.addExternalEndpoint( + RegistryTypeUtils.ipcEndpoint( + CustomRegistryConstants.AM_IPC_PROTOCOL, + true, + RegistryTypeUtils.marshall(rpcServiceAddress))); + // internal services - + - sliderAMProvider.applyInitialRegistryDefinitions(unsecureWebAPI, - secureWebAPI, - instanceData, - serviceRecord); ++ + sliderAMProvider.applyInitialRegistryDefinitions(amWebURI, + agentOpsURI, + agentStatusURI, - instanceData); ++ instanceData, ++ serviceRecord); // provider service dynamic definitions. - providerService.applyInitialRegistryDefinitions(unsecureWebAPI, - secureWebAPI, - instanceData, - serviceRecord); + providerService.applyInitialRegistryDefinitions(amWebURI, + agentOpsURI, + agentStatusURI, - instanceData); ++ instanceData, ++ serviceRecord); // push the registration info to ZK -- - registry.registerSelf(instanceData, unsecureWebAPI); + registry.registerSelf( + instanceData, amWebURI + ); + yarnRegistry.putServiceEntry(service_user_name, + SliderKeys.APP_TYPE, + instanceName, + serviceRecord ); + yarnRegistry.putServiceLiveness(service_user_name, + SliderKeys.APP_TYPE, + instanceName, + true, + true); + // and an ephemeral binding to the app + yarnRegistry.putComponent(service_user_name, + SliderKeys.APP_TYPE, + instanceName, + "appmaster", + serviceRecord, + true); + - } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy ---------------------------------------------------------------------- diff --cc slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy index 30b3655,32efbac..7c5de08 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy @@@ -245,10 -240,8 +245,11 @@@ class MockProviderService implements Pr @Override void applyInitialRegistryDefinitions( - URL unsecureWebAPI, - URL secureWebAPI, - URL amWebURI, URL agentOpsURI, URL agentStatusURI, ServiceInstanceData registryInstanceData) - throws MalformedURLException, IOException { ++ URL amWebURI, ++ URL agentOpsURI, ++ URL agentStatusURI, + ServiceInstanceData registryInstanceData, - ServiceRecord serviceEntry) throws IOException { ++ ServiceRecord serviceRecord) throws IOException { } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java ---------------------------------------------------------------------- diff --cc slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java index 631587a,dc11050..372038b --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java +++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java @@@ -22,8 -22,6 +22,7 @@@ package org.apache.slider.providers.hba import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.Container; - import org.apache.hadoop.yarn.registry.client.draft1.RegistryWriter; +import org.apache.hadoop.yarn.registry.client.types.ServiceRecord; import org.apache.slider.api.InternalKeys; import org.apache.slider.common.SliderKeys; import org.apache.slider.api.ClusterDescription; @@@ -247,15 -245,18 +246,6 @@@ public class HBaseProviderService exten } @Override - public void applyInitialRegistryDefinitions(URL web, - URL secureWebAPI, - ServiceInstanceData instanceData, ServiceRecord serviceRecord) throws - IOException { - super.applyInitialRegistryDefinitions(web, secureWebAPI, instanceData, - serviceRecord); - } - - @Override - public void applyInitialRegistryDefinitions(URL amWebURI, - URL agentOpsURI, - URL agentStatusURI, - ServiceInstanceData instanceData) throws - IOException { - super.applyInitialRegistryDefinitions(amWebURI, agentOpsURI, - agentStatusURI, - instanceData - ); - } - - @Override protected void serviceStart() throws Exception { registerHBaseServiceEntry(); super.serviceStart(); @@@ -277,24 -278,6 +267,7 @@@ configSet.put(HBASE_SITE_PUBLISHED_CONFIG, publishedSite); registry.registerServiceInstance(instanceData, null); + - ServiceRecord yse = new ServiceRecord(); - - - RegistryWriter registryWriter = yarnRegistry.getRegistryWriter(); - registryWriter.putServiceEntry( - yarnRegistry.getUser(), - HBASE_SERVICE_TYPE, - yarnRegistry.getInstanceName(), - yse); - - registryWriter.putServiceLiveness( - yarnRegistry.getUser(), - HBASE_SERVICE_TYPE, - yarnRegistry.getInstanceName(), - true, true); - - } /**