Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/434a964b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/434a964b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/434a964b
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 434a964b65fad17e8155b5f76f579ff675d32394
Parents: 5de46b8 7b7d29e
Author: Steve Loughran <ste...@apache.org>
Authored: Mon Aug 25 14:37:26 2014 +0100
Committer: Steve Loughran <ste...@apache.org>
Committed: Mon Aug 25 14:37:26 2014 +0100
----------------------------------------------------------------------
app-packages/accumulo/pom.xml | 2 +-
.../command-logger/application-pkg/pom.xml | 2 +-
.../command-logger/slider-pkg/appConfig.json | 2 -
.../command-logger/slider-pkg/metainfo.xml | 8 +
app-packages/command-logger/slider-pkg/pom.xml | 2 +-
app-packages/hbase/appConfig.json | 38 +----
app-packages/hbase/configuration/global.xml | 160 -------------------
app-packages/hbase/configuration/hbase-env.xml | 117 ++++++++++++++
.../hbase/configuration/hbase-log4j.xml | 1 +
app-packages/hbase/configuration/hbase-site.xml | 19 ++-
app-packages/hbase/get-hbase-site.sh | 15 ++
app-packages/hbase/metainfo.xml | 24 +++
app-packages/hbase/package/scripts/hbase.py | 26 +--
app-packages/hbase/package/scripts/params.py | 26 ++-
.../hbase/package/templates/hbase-env.sh.j2 | 81 ----------
app-packages/storm/appConfig.json | 88 +---------
app-packages/storm/configuration/global.xml | 39 -----
app-packages/storm/configuration/storm-env.xml | 40 +++++
app-packages/storm/configuration/storm-site.xml | 15 +-
app-packages/storm/metainfo.xml | 14 ++
app-packages/storm/package/scripts/params.py | 8 +-
app-packages/storm/package/scripts/storm.py | 7 +-
.../storm/package/scripts/yaml_config.py | 7 +-
pom.xml | 4 +-
slider-agent/pom.xml | 2 +-
.../src/main/python/agent/Controller.py | 2 +-
slider-agent/src/main/python/agent/Registry.py | 12 +-
slider-agent/src/main/python/agent/main.py | 7 +-
.../core/providers/__init__.py | 19 ++-
.../core/providers/system.py | 8 +
.../core/resources/system.py | 1 +
.../python/resource_management/TestPackage.py | 32 ++++
slider-assembly/pom.xml | 2 +-
slider-assembly/src/conf/slider-client.xml | 32 ++--
slider-assembly/src/main/scripts/slider | 3 +-
slider-assembly/src/main/scripts/slider.py | 21 ++-
slider-core/pom.xml | 2 +-
.../java/org/apache/slider/api/StatusKeys.java | 7 +-
.../registry/info/CustomRegistryConstants.java | 9 +-
.../providers/AbstractProviderService.java | 15 +-
.../slider/providers/ProviderService.java | 15 +-
.../providers/agent/AgentProviderService.java | 160 ++++++++++++++-----
.../slider/providers/agent/AgentUtils.java | 31 +++-
.../agent/application/metadata/Application.java | 11 +-
.../agent/application/metadata/Component.java | 6 +
.../agent/application/metadata/ConfigFile.java | 53 ++++++
.../metadata/ConfigurationDependencies.java | 39 -----
.../application/metadata/DefaultConfig.java | 39 +++++
.../metadata/DefaultConfigParser.java | 54 +++++++
.../application/metadata/MetainfoParser.java | 9 +-
.../application/metadata/PropertyInfo.java | 54 +++++++
.../slideram/SliderAMProviderService.java | 28 ++--
.../server/appmaster/SliderAppMaster.java | 49 +++---
.../server/appmaster/web/AgentService.java | 28 +---
.../appmaster/web/rest/agent/AgentWebApp.java | 16 +-
.../server/services/security/SecurityUtils.java | 60 ++++---
.../LaunchedWorkflowCompositeService.java | 4 +
.../services/workflow/ClosingService.java | 3 +
.../test_command_log/appConfig.json | 2 -
.../test_command_log/appConfig_fast_no_reg.json | 2 -
.../test_command_log/appConfig_no_hb.json | 2 -
.../agent/AgentMiniClusterTestBase.groovy | 9 +-
.../common/tools/TestWindowsSupport.groovy | 112 +++++++++++++
.../agent/TestAgentAMManagementWS.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 7 +-
.../publisher/TestPublisherRestResources.groovy | 11 +-
.../apache/slider/test/SliderTestUtils.groovy | 39 +++++
.../slider/test/YarnMiniClusterTestBase.groovy | 23 +--
.../agent/TestAgentProviderService.java | 101 ++++++++++--
.../application/metadata/TestConfigParser.java | 107 +++++++++++++
.../metadata/TestMetainfoParser.java | 7 +
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../workflow/TestWorkflowClosingService.java | 4 +-
.../slider/common/tools/test/metainfo.xml | 11 +-
slider-funtest/pom.xml | 2 +-
.../accumulo/accumulo-funtests/pom.xml | 2 +-
.../accumulo/slider-accumulo-provider/pom.xml | 2 +-
.../providers/accumulo/AccumuloTestBase.groovy | 7 +-
slider-providers/hbase/hbase-funtests/pom.xml | 2 +-
.../hbase/slider-hbase-provider/pom.xml | 2 +-
.../providers/hbase/HBaseProviderService.java | 27 ----
.../minicluster/HBaseMiniClusterTestBase.groovy | 9 +-
82 files changed, 1296 insertions(+), 777 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --cc
slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 89e5c65,baddb56..0e7ea25
---
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@@ -335,15 -331,13 +335,16 @@@ public abstract class AbstractProviderS
}
}
}
+
@Override
- public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData registryInstanceData,
- ServiceRecord serviceRecord)
- throws IOException {
-
- this.amWebAPI = unsecureWebAPI;
+ public void applyInitialRegistryDefinitions(URL amWebURI,
+ URL agentOpsURI,
+ URL agentStatusURI,
- ServiceInstanceData
registryInstanceData) throws IOException {
++ ServiceInstanceData
registryInstanceData,
++ ServiceRecord serviceRecord)
++ throws IOException {
+
+ this.amWebAPI = amWebURI;
this.registryInstanceData = registryInstanceData;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --cc
slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 67ca0a9,a9aebad..b28ac6a
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@@ -184,15 -176,16 +184,18 @@@ public interface ProviderService extend
/**
* Prior to going live -register the initial service registry data
- * @param unsecureWebAPI
- * @param secureWebAPI
+ * @param amWebURI
+ * @param agentOpsURI
+ * @param agentStatusURI
* @param registryInstanceData
+ * @param serviceRecord
*/
- void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData registryInstanceData,
- ServiceRecord serviceRecord) throws IOException;
+ void applyInitialRegistryDefinitions(URL amWebURI,
+ URL agentOpsURI,
+ URL agentStatusURI,
- ServiceInstanceData
registryInstanceData)
++ ServiceInstanceData
registryInstanceData,
++ ServiceRecord serviceRecord)
+ throws IOException;
/**
* Create the container release selector for this provider...any policy
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --cc
slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 790e379,e79fb60..f78d468
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@@ -585,29 -600,25 +603,40 @@@ public class AgentProviderService exten
}
@Override
- public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData instanceData, ServiceRecord serviceRecord) throws
- IOException {
- super.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData,
- serviceRecord);
- public void applyInitialRegistryDefinitions(URL amWebURI,
- URL agentOpsURI,
- URL agentStatusURI,
- ServiceInstanceData
instanceData) throws IOException {
++ public void applyInitialRegistryDefinitions (URL amWebURI,
++ URL agentOpsURI,
++ URL agentStatusURI,
++ ServiceInstanceData instanceData,
++ ServiceRecord serviceRecord)
++ throws IOException {
+ super.applyInitialRegistryDefinitions(amWebURI,
+ agentOpsURI,
- agentStatusURI, instanceData);
++ agentStatusURI,
++ instanceData,
++ serviceRecord);
try {
- URL url = new URL(secureWebAPI, SLIDER_PATH_AGENTS);
++ URL restURL = new URL(agentOpsURI, SLIDER_PATH_AGENTS);
+
- URI uri = url.toURI();
instanceData.internalView.endpoints.put(
- CustomRegistryConstants.AGENT_REST_API,
+ CustomRegistryConstants.AGENT_SECURE_REST_API,
- new RegisteredEndpoint(
- new URL(agentOpsURI, SLIDER_PATH_AGENTS),
++ new RegisteredEndpoint(restURL,
+ "Agent Secure REST API"));
++ URL agentStatusURL = new URL(agentStatusURI, SLIDER_PATH_AGENTS);
+ instanceData.internalView.endpoints.put(
+ CustomRegistryConstants.AGENT_ONEWAY_REST_API,
new RegisteredEndpoint(
- url,
- "Agent REST API"));
- new URL(agentStatusURI, SLIDER_PATH_AGENTS),
++ agentStatusURL,
+ "Agent Oneway REST API"));
+
+ serviceRecord.addInternalEndpoint(
- new Endpoint(CustomRegistryConstants.AGENT_REST_API,
++ new Endpoint(CustomRegistryConstants.AGENT_SECURE_REST_API,
++ ProtocolTypes.PROTOCOL_REST,
++ restURL.toURI()));
++ serviceRecord.addInternalEndpoint(
++ new Endpoint(CustomRegistryConstants.AGENT_ONEWAY_REST_API,
+ ProtocolTypes.PROTOCOL_REST,
- uri));
++ agentStatusURL.toURI()));
} catch (URISyntaxException e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --cc
slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index ab41549,863ea7e..2b62d24
---
a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@@ -112,15 -110,15 +112,17 @@@ public class SliderAMProviderService ex
}
@Override
- public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
+ public void applyInitialRegistryDefinitions(URL amWebURI,
- URL agentOpsURI,
- URL agentStatusURI,
- ServiceInstanceData
instanceData) throws IOException {
++ URL agentOpsURI,
++ URL agentStatusURI,
+ ServiceInstanceData instanceData,
- ServiceRecord serviceRecord) throws IOException {
- super.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData,
++ ServiceRecord serviceRecord)
++ throws IOException {
+ super.applyInitialRegistryDefinitions(amWebURI,
- agentOpsURI,
- agentStatusURI, instanceData
- );
-
++ agentOpsURI,
++ agentStatusURI,
++ instanceData,
+ serviceRecord);
-
// now publish site.xml files
YarnConfiguration defaultYarnConfig = new YarnConfiguration();
amState.getPublishedSliderConfigurations().put(
@@@ -151,15 -149,10 +153,14 @@@
try {
- RegistryView externalView = instanceData.externalView;
RegisteredEndpoint webUI =
- new RegisteredEndpoint(unsecureWebAPI, "Application Master Web UI");
+ new RegisteredEndpoint(amWebURI, "Application Master Web UI");
- URL managementAPI = new URL(unsecureWebAPI, SLIDER_PATH_MANAGEMENT);
- URL registryREST = new URL(unsecureWebAPI,
RestPaths.SLIDER_PATH_REGISTRY + "/" +
++ URL managementAPI = new URL(amWebURI, SLIDER_PATH_MANAGEMENT);
++ URL registryREST = new URL(amWebURI, RestPaths.SLIDER_PATH_REGISTRY +
"/" +
+ RestPaths.REGISTRY_SERVICE);
- URL publisherURL = new URL(unsecureWebAPI, SLIDER_PATH_PUBLISHER);
+
+ RegistryView externalView = instanceData.externalView;
externalView.endpoints.put(CommonRegistryConstants.WEB_UI, webUI);
externalView.endpoints.put(
@@@ -171,9 -164,11 +172,10 @@@
externalView.endpoints.put(
CustomRegistryConstants.REGISTRY_REST_API,
new RegisteredEndpoint(
- new URL(amWebURI, RestPaths.SLIDER_PATH_REGISTRY + "/" +
- RestPaths.REGISTRY_SERVICE),
+ registryREST,
"Registry Web Service" ) );
+ URL publisherURL = new URL(amWebURI, SLIDER_PATH_PUBLISHER);
externalView.endpoints.put(
CustomRegistryConstants.PUBLISHER_REST_API,
new RegisteredEndpoint(
@@@ -185,20 -181,6 +187,18 @@@
externalView.configurationsURL = SliderUtils.appendToURL(
publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET);
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.webEndpoint(
- CommonRegistryConstants.WEB_UI, unsecureWebAPI.toURI()));
++ CommonRegistryConstants.WEB_UI, amWebURI.toURI()));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.MANAGEMENT_REST_API,
+ managementAPI.toURI()));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.PUBLISHER_REST_API,
+ publisherURL.toURI()));
-
-
+
} catch (URISyntaxException e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --cc
slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index a387d3a,b4515f1..c743812
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@@ -871,11 -857,12 +876,12 @@@ public class SliderAppMaster extends Ab
private void registerServiceInstance(String instanceName,
ApplicationId appid) throws Exception {
// the registry is running, so register services
- URL unsecureWebAPI = new URL(appMasterTrackingUrl);
- URL secureWebAPI = new URL(agentAccessUrl);
+ URL amWebURI = new URL(appMasterTrackingUrl);
+ URL agentOpsURI = new URL(agentOpsUrl);
+ URL agentStatusURI = new URL(agentStatusUrl);
String serviceName = SliderKeys.APP_TYPE;
int id = appid.getId();
- String appServiceType = RegistryNaming.createRegistryServiceType(
+ String serviceType = RegistryNaming.createRegistryServiceType(
instanceName,
service_user_name,
serviceName);
@@@ -907,52 -883,28 +913,55 @@@
CustomRegistryConstants.AM_IPC_PROTOCOL,
new RegisteredEndpoint(rpcServiceAddress,
RegisteredEndpoint.PROTOCOL_HADOOP_PROTOBUF,
- "Slider AM RPC") );
+ SLIDER_AM_RPC) );
+ // Yarn registry
+ ServiceRecord serviceRecord = new ServiceRecord();
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.ipcEndpoint(
+ CustomRegistryConstants.AM_IPC_PROTOCOL,
+ true,
+ RegistryTypeUtils.marshall(rpcServiceAddress)));
+
// internal services
-
+
- sliderAMProvider.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData,
- serviceRecord);
++
+ sliderAMProvider.applyInitialRegistryDefinitions(amWebURI,
+ agentOpsURI,
+ agentStatusURI,
- instanceData);
++ instanceData,
++ serviceRecord);
// provider service dynamic definitions.
- providerService.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData,
- serviceRecord);
+ providerService.applyInitialRegistryDefinitions(amWebURI,
+ agentOpsURI,
+ agentStatusURI,
- instanceData);
++ instanceData,
++ serviceRecord);
// push the registration info to ZK
--
- registry.registerSelf(instanceData, unsecureWebAPI);
+ registry.registerSelf(
+ instanceData, amWebURI
+ );
+ yarnRegistry.putServiceEntry(service_user_name,
+ SliderKeys.APP_TYPE,
+ instanceName,
+ serviceRecord );
+ yarnRegistry.putServiceLiveness(service_user_name,
+ SliderKeys.APP_TYPE,
+ instanceName,
+ true,
+ true);
+ // and an ephemeral binding to the app
+ yarnRegistry.putComponent(service_user_name,
+ SliderKeys.APP_TYPE,
+ instanceName,
+ "appmaster",
+ serviceRecord,
+ true);
+
-
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --cc
slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 30b3655,32efbac..7c5de08
---
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@@ -245,10 -240,8 +245,11 @@@ class MockProviderService implements Pr
@Override
void applyInitialRegistryDefinitions(
- URL unsecureWebAPI,
- URL secureWebAPI,
- URL amWebURI, URL agentOpsURI, URL agentStatusURI,
ServiceInstanceData registryInstanceData)
- throws MalformedURLException, IOException {
++ URL amWebURI,
++ URL agentOpsURI,
++ URL agentStatusURI,
+ ServiceInstanceData registryInstanceData,
- ServiceRecord serviceEntry) throws IOException {
++ ServiceRecord serviceRecord) throws IOException {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/434a964b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --cc
slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 631587a,dc11050..372038b
---
a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++
b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@@ -22,8 -22,6 +22,7 @@@ package org.apache.slider.providers.hba
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
- import org.apache.hadoop.yarn.registry.client.draft1.RegistryWriter;
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.api.ClusterDescription;
@@@ -247,15 -245,18 +246,6 @@@ public class HBaseProviderService exten
}
@Override
- public void applyInitialRegistryDefinitions(URL web,
- URL secureWebAPI,
- ServiceInstanceData instanceData, ServiceRecord serviceRecord) throws
- IOException {
- super.applyInitialRegistryDefinitions(web, secureWebAPI, instanceData,
- serviceRecord);
- }
-
- @Override
- public void applyInitialRegistryDefinitions(URL amWebURI,
- URL agentOpsURI,
- URL agentStatusURI,
- ServiceInstanceData
instanceData) throws
- IOException {
- super.applyInitialRegistryDefinitions(amWebURI, agentOpsURI,
- agentStatusURI,
- instanceData
- );
- }
-
- @Override
protected void serviceStart() throws Exception {
registerHBaseServiceEntry();
super.serviceStart();
@@@ -277,24 -278,6 +267,7 @@@
configSet.put(HBASE_SITE_PUBLISHED_CONFIG, publishedSite);
registry.registerServiceInstance(instanceData, null);
+
- ServiceRecord yse = new ServiceRecord();
-
-
- RegistryWriter registryWriter = yarnRegistry.getRegistryWriter();
- registryWriter.putServiceEntry(
- yarnRegistry.getUser(),
- HBASE_SERVICE_TYPE,
- yarnRegistry.getInstanceName(),
- yse);
-
- registryWriter.putServiceLiveness(
- yarnRegistry.getUser(),
- HBASE_SERVICE_TYPE,
- yarnRegistry.getInstanceName(),
- true, true);
-
-
}
/**
