SLIDER-149 YARN and slider registries living side by side, though YARN reg is doing simple registration only
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7e1c77f8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7e1c77f8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7e1c77f8 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 7e1c77f891fa797f12b282417c1bb8b21d6c4e18 Parents: 96f4acb Author: Steve Loughran <ste...@apache.org> Authored: Mon Aug 4 14:03:02 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Aug 4 14:03:02 2014 +0100 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 26 ++++++++++++++++++-- .../server/appmaster/SliderAppMaster.java | 2 ++ .../server/services/curator/CuratorHelper.java | 5 ++++ .../server/services/curator/CuratorService.java | 17 +++++++++++++ .../utility/AbstractSliderLaunchedService.java | 6 ++--- .../standalone/TestStandaloneRegistryAM.groovy | 5 ++++ 6 files changed, 56 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 6b67342..87af625 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.registry.client.api.RegistryConstants; +import org.apache.hadoop.yarn.registry.client.binding.zk.ZKPathDumper; import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; @@ -666,7 +667,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public FsPermission getClusterDirectoryPermissions(Configuration conf) { String clusterDirPermsOct = conf.get(CLUSTER_DIRECTORY_PERMISSIONS, - DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS); + DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS); return new FsPermission(clusterDirPermsOct); } @@ -1233,7 +1234,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe */ private boolean getUsingMiniMRCluster() { return getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, - false); + false); } /** @@ -2334,6 +2335,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return getRegistry().listInstances(SliderKeys.APP_TYPE); } + + /** + * Get an on-demand path jumper + * @return a class that can dump the contents of the registry + */ + @VisibleForTesting + public ZKPathDumper dumpSliderRegistry() throws SliderException, IOException { + return getRegistry().dumpPath(); + } + /** * List instances in the registry * @return the instance IDs @@ -2410,6 +2421,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return maybeStartYarnRegistry(); } + + /** + * Get an on-demand path jumper + * @return a class that can dump the contents of the registry + */ + @VisibleForTesting + public ZKPathDumper dumpYarnRegistry() throws SliderException, IOException { + return getYarnRegistry().dumpPath(); + } + + /** * Output to standard out/stderr (implementation specific detail) * @param src source http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 87c5295..236c5b6 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -603,6 +603,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService //registry log.info("Starting slider registry"); sliderRegistry = startRegistrationService(); + log.info(sliderRegistry.toString()); + log.info("Starting Yarn registry"); yarnRegistry = startYarnRegistryService(); log.info(yarnRegistry.toString()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java index a6316d5..72d413f 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java @@ -135,4 +135,9 @@ public class CuratorHelper extends Configured { ServiceInstanceData.class); } + + @Override + public String toString() { + return "Curator binding " + connectionString; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java index b5b0c14..cfdf80d 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java @@ -18,12 +18,14 @@ package org.apache.slider.server.services.curator; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.utils.ZKPaths; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.registry.client.binding.zk.ZKPathDumper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,4 +97,19 @@ public class CuratorService extends AbstractService { public final CuratorHelper getCuratorHelper() { return curatorHelper; } + + @Override + public String toString() { + return super.toString() + "; " + + (curatorHelper != null ? curatorHelper : "( unbound)") + + "; " + basePath; + } + + /** + * Get an on-demand path jumper + * @return a class that can dump the contents of the registry + */ + public ZKPathDumper dumpPath() { + return new ZKPathDumper(curator, basePath); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java index d059f34..4b78d52 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java @@ -65,9 +65,9 @@ public abstract class AbstractSliderLaunchedService extends throws BadConfigException { String registryQuorum = lookupZKQuorum(); - String zkPath = getConfig().get(RegistryConstants.REGISTRY_ZK_ROOT, - RegistryConstants.DEFAULT_REGISTRY_ROOT); - zkPath = getConfig().get(REGISTRY_PATH, zkPath); + String zkPath = getConfig().get( + SliderXmlConfKeys.REGISTRY_PATH, + SliderXmlConfKeys.DEFAULT_REGISTRY_PATH); return startSliderRegistrationService(registryQuorum, zkPath); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy index 4806adb..7e8ea02 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy @@ -111,6 +111,11 @@ class TestStandaloneRegistryAM extends AgentMiniClusterTestBase { //switch to the ZK-based registry + + log.info("slider service registry: \n${client.dumpSliderRegistry()}\n") + log.info("yarn service registry: \n${client.dumpYarnRegistry()}\n") + + describe "service registry names" SliderRegistryService registryService = client.registry def serviceTypes = registryService.serviceTypes;
