SLIDER-149; in sync with registry code. HBase provider test working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/46fb5dbd Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/46fb5dbd Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/46fb5dbd Branch: refs/heads/develop Commit: 46fb5dbdb35c5b072771a394ec339f1bd7e8f56c Parents: 2154c75 Author: Steve Loughran <ste...@apache.org> Authored: Fri Sep 26 14:18:06 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Sep 26 14:18:06 2014 +0100 ---------------------------------------------------------------------- .../registry/retrieve/RegistryRetriever.java | 43 +++++++++++++------- .../server/appmaster/SliderAppMaster.java | 3 +- .../YarnRegistryViewForProviders.java | 17 +++++--- .../TestStandaloneYarnRegistryAM.groovy | 2 +- .../providers/hbase/HBaseProviderService.java | 12 ++++-- 5 files changed, 51 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java index 1d11ef3..df1d675 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java @@ -25,8 +25,10 @@ import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.json.JSONConfiguration; +import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils; import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException; +import org.apache.hadoop.yarn.registry.client.exceptions.RegistryIOException; import org.apache.hadoop.yarn.registry.client.types.Endpoint; import org.apache.hadoop.yarn.registry.client.types.ServiceRecord; import org.apache.slider.common.tools.SliderUtils; @@ -68,25 +70,37 @@ public class RegistryRetriever { this.internalConfigurationURL = internalConfigurationURL; } - public RegistryRetriever(ServiceRecord record) throws InvalidRecordException { + /** + * Retrieve from a service by locating the + * exported {@link CustomRegistryConstants#PUBLISHER_CONFIGURATIONS_API} + * and working off it. + * @param record service record + * @throws RegistryIOException the address type of the endpoint does + * not match that expected (i.e. not a list of URLs), missing endpoint... + */ + public RegistryRetriever(ServiceRecord record) throws RegistryIOException { Endpoint internal = record.getInternalEndpoint( CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API); - List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType( - internal); - if (addresses != null && !addresses.isEmpty()) { - internalConfigurationURL = addresses.get(0); - } else { - internalConfigurationURL = ""; + String url = null; + if (internal != null) { + List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType( + internal); + if (addresses != null && !addresses.isEmpty()) { + url = addresses.get(0); + } } + internalConfigurationURL = url; Endpoint external = record.getExternalEndpoint( CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API); - - addresses = RegistryTypeUtils.retrieveAddressesUriType(external); - if (addresses != null && !addresses.isEmpty()) { - externalConfigurationURL = addresses.get(0); - } else { - externalConfigurationURL = ""; + url = null; + if (external != null) { + List<String> addresses = + RegistryTypeUtils.retrieveAddressesUriType(external); + if (addresses != null && !addresses.isEmpty()) { + url = addresses.get(0); + } } + externalConfigurationURL = url; } /** @@ -166,7 +180,8 @@ public class RegistryRetriever { @Override public String toString() { return super.toString() - + " - external " + externalConfigurationURL; + + ": internal URL: \"" + internalConfigurationURL + + "\"; external \"" + externalConfigurationURL +"\""; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 2d3e5e4..2625d74 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -901,7 +901,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService yarnRegistryOperations = new YarnRegistryViewForProviders( registryOperations, service_user_name, SliderKeys.APP_TYPE, - instanceName); + instanceName, + appAttemptID); providerService.bindToYarnRegistry(yarnRegistryOperations); sliderAMProvider.bindToYarnRegistry(yarnRegistryOperations); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java index d699279..e5b150a 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java @@ -18,6 +18,7 @@ package org.apache.slider.server.services.yarnregistry; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.registry.client.api.RegistryOperations; import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils; import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils; @@ -35,14 +36,23 @@ public class YarnRegistryViewForProviders { private final String sliderServiceclass; private final String instanceName; + private final ApplicationAttemptId applicationAttemptId; private ServiceRecord selfRegistration; public YarnRegistryViewForProviders(RegistryOperations registryOperations, - String user, String sliderServiceclass, String instanceName) { + String user, + String sliderServiceclass, + String instanceName, + ApplicationAttemptId applicationAttemptId) { this.registryOperations = registryOperations; this.user = user; this.sliderServiceclass = sliderServiceclass; this.instanceName = instanceName; + this.applicationAttemptId = applicationAttemptId; + } + + public ApplicationAttemptId getApplicationAttemptId() { + return applicationAttemptId; } public String getUser() { @@ -100,7 +110,6 @@ public class YarnRegistryViewForProviders { registryOperations.create(path, record, CreateFlags.OVERWRITE); } - /** * Add a service under a path * @param username user @@ -117,10 +126,8 @@ public class YarnRegistryViewForProviders { username, serviceClass, serviceName); registryOperations.mknode(RegistryPathUtils.parentOf(path), true); registryOperations.create(path, record, CreateFlags.OVERWRITE); - } - /** * Add a service under a path for the current user * @param serviceClass service class to use under ~user @@ -136,10 +143,8 @@ public class YarnRegistryViewForProviders { user, serviceClass, serviceName); registryOperations.mknode(RegistryPathUtils.parentOf(path), true); registryOperations.create(path, record, CreateFlags.OVERWRITE); - } - public void rmComponent(String componentName) throws IOException { String path = RegistryOperationUtils.componentPath( user, sliderServiceclass, instanceName, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy index 4cd662b..d081676 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy @@ -139,7 +139,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { RegistryOperationsClient registryOperationsClient = registryService as RegistryOperationsClient try { - def yarnRegistryDump = registryOperationsClient.dumpPath() + def yarnRegistryDump = registryOperationsClient.dumpPath(false) log.info("yarn service registry: \n${yarnRegistryDump}\n") } catch (IOException ignored) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java index 3c059d9..1760e72 100644 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java +++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils; +import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies; import org.apache.hadoop.yarn.registry.client.types.ServiceRecord; import org.apache.slider.api.InternalKeys; import org.apache.slider.common.SliderKeys; @@ -254,16 +255,19 @@ public class HBaseProviderService extends AbstractProviderService private void registerHBaseServiceEntry() throws IOException { - String name = amState.getApplicationName() ; ServiceRecord serviceRecord = new ServiceRecord(); - + // bond lifespan to the application + serviceRecord.yarn_id = yarnRegistry.getApplicationAttemptId() + .getApplicationId().toString(); + serviceRecord.yarn_persistence = PersistencePolicies.APPLICATION; try { - URL configURL = new URL(amWebAPI, SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE); + URL configURL = new URL(amWebAPI, + SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE); serviceRecord.addExternalEndpoint( RegistryTypeUtils.restEndpoint( - CustomRegistryConstants.PUBLISHER_REST_API, + CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API, configURL.toURI())); } catch (URISyntaxException e) { log.warn("failed to create config URL: {}", e, e);