YARN-7198. Add jsvc support for RegistryDNS. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92734800 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92734800 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92734800 Branch: refs/heads/trunk Commit: 92734800844b93bb19f3c0fca6be88d2801f7b1b Parents: b57144a Author: Jian He <jia...@apache.org> Authored: Fri Oct 13 10:49:38 2017 -0700 Committer: Jian He <jia...@apache.org> Committed: Mon Nov 6 13:30:18 2017 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/hadoop-yarn/bin/yarn | 6 ++ .../hadoop-yarn/conf/yarn-env.sh | 12 +++ .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 73 +++++++++++++++++- .../dns/PrivilegedRegistryDNSStarter.java | 80 ++++++++++++++++++++ .../hadoop/registry/server/dns/RegistryDNS.java | 45 +++++++---- .../registry/server/dns/RegistryDNSServer.java | 57 +++++--------- .../src/site/markdown/YarnCommands.md | 8 ++ .../src/site/markdown/registry/index.md | 1 + .../markdown/registry/registry-configuration.md | 4 +- .../site/markdown/yarn-service/RegistryDNS.md | 40 +++++++++- .../markdown/yarn-service/ServiceDiscovery.md | 21 ++++- 11 files changed, 282 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 05d892b..727384c 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -45,6 +45,7 @@ function hadoop_usage hadoop_add_subcommand "nodemanager" daemon "run a nodemanager on each worker" hadoop_add_subcommand "proxyserver" daemon "run the web app proxy server" hadoop_add_subcommand "queue" client "prints queue information" + hadoop_add_subcommand "registrydns" daemon "run the registry DNS server" hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager" hadoop_add_subcommand "rmadmin" admin "admin tools" hadoop_add_subcommand "router" daemon "run the Router daemon" @@ -143,6 +144,11 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" queue) HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI ;; + registrydns) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_SECURE_CLASSNAME='org.apache.hadoop.registry.server.dns.PrivilegedRegistryDNSStarter' + HADOOP_CLASSNAME='org.apache.hadoop.registry.server.dns.RegistryDNSServer' + ;; resourcemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh index 90a87bf..4bd1d3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh +++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh @@ -160,3 +160,15 @@ # See ResourceManager for some examples # #export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')" + +### +# Registry DNS specific parameters +### +# For privileged registry DNS, user to run as after dropping privileges +# This will replace the hadoop.id.str Java property in secure mode. +# export YARN_REGISTRYDNS_SECURE_USER=yarn + +# Supplemental options for privileged registry DNS +# By default, Hadoop uses jsvc which needs to know to launch a +# server jvm. +# export YARN_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server" http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml index e083312..4e805cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml @@ -34,6 +34,21 @@ <dependencies> <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-annotations</artifactId> + </dependency> + + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-api</artifactId> </dependency> @@ -70,14 +85,68 @@ </dependency> <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-client</artifactId> + </dependency> + + <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-framework</artifactId> </dependency> <dependency> <groupId>org.apache.curator</groupId> - <artifactId>curator-test</artifactId> - <scope>test</scope> + <artifactId>curator-recipes</artifactId> + </dependency> + + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + </dependency> + + <dependency> + <groupId>commons-daemon</groupId> + <artifactId>commons-daemon</artifactId> + </dependency> + + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + + <dependency> + <groupId>commons-net</groupId> + <artifactId>commons-net</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> + + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/PrivilegedRegistryDNSStarter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/PrivilegedRegistryDNSStarter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/PrivilegedRegistryDNSStarter.java new file mode 100644 index 0000000..dd4e1b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/PrivilegedRegistryDNSStarter.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.registry.server.dns; + +import org.apache.commons.daemon.Daemon; +import org.apache.commons.daemon.DaemonContext; +import org.apache.hadoop.registry.client.api.DNSOperationsFactory; +import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.registry.client.api.RegistryConstants.DEFAULT_DNS_PORT; +import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_PORT; + +/** + * This class is used to allow the RegistryDNSServer to run on a privileged + * port (e.g. 53). + */ +public class PrivilegedRegistryDNSStarter implements Daemon { + private static final Logger LOG = + LoggerFactory.getLogger(PrivilegedRegistryDNSStarter.class); + + private YarnConfiguration conf; + private RegistryDNS registryDNS; + private RegistryDNSServer registryDNSServer; + + @Override + public void init(DaemonContext context) throws Exception { + String[] args = context.getArguments(); + StringUtils.startupShutdownMessage(RegistryDNSServer.class, args, LOG); + conf = new YarnConfiguration(); + new GenericOptionsParser(conf, args); + + int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT); + if (port < 1 || port > 1023) { + throw new RuntimeException("Must start privileged registry DNS server " + + "with '" + KEY_DNS_PORT + "' configured to a privileged port."); + } + + try { + registryDNS = (RegistryDNS) DNSOperationsFactory.createInstance(conf); + registryDNS.initializeChannels(conf); + } catch (Exception e) { + LOG.error("Error initializing Registry DNS", e); + throw e; + } + } + + @Override + public void start() throws Exception { + registryDNSServer = RegistryDNSServer.launchDNSServer(conf, registryDNS); + } + + @Override + public void stop() throws Exception { + } + + @Override + public void destroy() { + registryDNSServer.stop(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java index 51139be..d7a415d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java @@ -82,6 +82,7 @@ import java.util.Calendar; import java.util.Collection; import java.util.Date; import java.util.Iterator; +import java.util.Map; import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; @@ -130,6 +131,8 @@ public class RegistryDNS extends AbstractService implements DNSOperations, private ConcurrentMap<Name, Zone> zones = new ConcurrentHashMap<>(); private Name bindHost; + private boolean channelsInitialized = false; + /** * Construct the service. * @@ -150,6 +153,24 @@ public class RegistryDNS extends AbstractService implements DNSOperations, }); } + public void initializeChannels(Configuration conf) throws Exception { + if (channelsInitialized) { + return; + } + channelsInitialized = true; + int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT); + InetAddress addr = InetAddress.getLocalHost(); + + String bindAddress = conf.get(KEY_DNS_BIND_ADDRESS); + if (bindAddress != null) { + addr = InetAddress.getByName(bindAddress); + } + + LOG.info("Opening TCP and UDP channels on {} port {}", addr, port); + addNIOUDP(addr, port); + addNIOTCP(addr, port); + } + /** * Initializes the registry. * @@ -164,17 +185,9 @@ public class RegistryDNS extends AbstractService implements DNSOperations, try { setDomainName(conf); - int port = initializeZones(conf); - - InetAddress addr = InetAddress.getLocalHost(); - - String bindAddress = conf.get(KEY_DNS_BIND_ADDRESS); - if (bindAddress != null) { - addr = InetAddress.getByName(bindAddress); - } - addNIOUDP(addr, port); - addNIOTCP(addr, port); + initializeZones(conf); + initializeChannels(conf); } catch (IOException e) { LOG.error("Error initializing Registry DNS Server", e); throw e; @@ -189,8 +202,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations, * @return the listener port * @throws IOException */ - int initializeZones(Configuration conf) throws IOException { - int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT); + void initializeZones(Configuration conf) throws IOException { ttl = conf.getTimeDuration(KEY_DNS_TTL, 1L, TimeUnit.SECONDS); RecordCreatorFactory.setTtl(ttl); @@ -203,7 +215,12 @@ public class RegistryDNS extends AbstractService implements DNSOperations, initializeReverseLookupZone(conf); - return port; + StringBuilder builder = new StringBuilder(); + builder.append("DNS zones: ").append(System.lineSeparator()); + for (Map.Entry<Name, Zone> entry : zones.entrySet()) { + builder.append(System.lineSeparator()).append(entry.getValue()); + } + LOG.info(builder.toString()); } /** @@ -1412,7 +1429,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations, } processor.manageDNSRecords(command); } else { - LOG.warn("Yarn Resgistry record {} does not contain {} attribute ", + LOG.warn("Yarn Registry record {} does not contain {} attribute ", record.toString(), YarnRegistryAttributes.YARN_PERSISTENCE); } } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java index faa5fe1..c7f6831 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java @@ -17,11 +17,6 @@ package org.apache.hadoop.registry.server.dns; import com.google.common.base.Preconditions; -import org.apache.commons.cli.BasicParser; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.registry.client.api.DNSOperationsFactory; @@ -65,9 +60,11 @@ public class RegistryDNSServer extends CompositeService { /** * Creates the DNS server. * @param name the server name. + * @param registryDNS the registry DNS instance. */ - public RegistryDNSServer(String name) { + public RegistryDNSServer(String name, final RegistryDNS registryDNS) { super(name); + this.registryDNS = registryDNS; } /** @@ -83,8 +80,9 @@ public class RegistryDNSServer extends CompositeService { registryOperations = new RegistryOperationsService("RegistryDNSOperations"); addService(registryOperations); - // probably need to populate with existing apps? - registryDNS = (RegistryDNS) DNSOperationsFactory.createInstance(conf); + if (registryDNS == null) { + registryDNS = (RegistryDNS) DNSOperationsFactory.createInstance(conf); + } addService(registryDNS); super.serviceInit(conf); @@ -231,24 +229,21 @@ public class RegistryDNSServer extends CompositeService { /** * Launch the server. - * @param args command line args. + * @param conf configuration + * @param rdns registry dns instance * @return */ - static RegistryDNSServer launchDNSServer(String[] args) { + static RegistryDNSServer launchDNSServer(Configuration conf, + RegistryDNS rdns) { RegistryDNSServer dnsServer = null; Thread .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); - StringUtils.startupShutdownMessage(RegistryDNSServer.class, args, - LOG); try { - dnsServer = new RegistryDNSServer("RegistryDNSServer"); + dnsServer = new RegistryDNSServer("RegistryDNSServer", rdns); ShutdownHookManager.get().addShutdownHook( new CompositeService.CompositeServiceShutdownHook(dnsServer), SHUTDOWN_HOOK_PRIORITY); - YarnConfiguration conf = new YarnConfiguration(); - processCommandLine(args, conf); - new GenericOptionsParser(conf, args); dnsServer.init(conf); dnsServer.start(); } catch (Throwable t) { @@ -259,32 +254,14 @@ public class RegistryDNSServer extends CompositeService { } /** - * Process input command line arguments. - * @param args the command line argument array. - * @param conf the configuration. - */ - private static void processCommandLine(String[] args, - YarnConfiguration conf) { - Options options = new Options(); - options.addOption("p", "port", true, - "the server listening port (override)"); - - CommandLineParser parser = new BasicParser(); - try { - CommandLine cmd = parser.parse(options, args); - if (cmd.hasOption("p")) { - conf.set(RegistryConstants.KEY_DNS_PORT, cmd.getOptionValue("p")); - } - } catch (ParseException e) { - LOG.error("Error parsing the command line options", e); - } - } - - /** * Lanches the server instance. * @param args the command line args. + * @throws IOException if command line options can't be parsed */ - public static void main(String[] args) { - launchDNSServer(args); + public static void main(String[] args) throws IOException { + StringUtils.startupShutdownMessage(RegistryDNSServer.class, args, LOG); + YarnConfiguration conf = new YarnConfiguration(); + new GenericOptionsParser(conf, args); + launchDNSServer(conf, null); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 51e27cc..6f00c5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -329,9 +329,17 @@ Usage: `yarn timelineserver` Start the TimeLineServer ### apiserver + Usage: `yarn apiserver` + Start the API-server for deploying/managing services on YARN +### registrydns + +Usage: `yarn registrydns` + +Start the RegistryDNS server + Files ----- http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md index 3a648b6..0a127cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md @@ -22,6 +22,7 @@ to allow deployed applications to register themselves and the means of communicating with them. Client applications can then locate services and use the binding information to connect with the services's network-accessible endpoints, be they REST, IPC, Web UI, Zookeeper quorum+path or some other protocol. +Currently, all the registry data is stored in a zookeeper cluster. * [Architecture](yarn-registry.html) * [Configuration](registry-configuration.html) http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md index adca451..46bc92d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md @@ -90,7 +90,7 @@ and the ports on which the ZK services are listening. ``` <property> <description> - List of hostname:port pairs defining the + A comma separated list of hostname:port pairs defining the zookeeper quorum binding for the registry </description> <name>hadoop.registry.zk.quorum</name> @@ -339,7 +339,7 @@ concluding that the quorum is unreachable and failing. <property> <description> - List of hostname:port pairs defining the + A comma separated list of hostname:port pairs defining the zookeeper quorum binding for the registry </description> <name>hadoop.registry.zk.quorum</name> http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md index ef395fc..c7f19ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md @@ -15,7 +15,7 @@ # Registry DNS Server <!-- MACRO{toc|fromDepth=0|toDepth=3} --> - +The document describes the internals of Registry DNS server. It is based on the [YARN service registry](../registry/index.md) which is backed by a zookeeper cluster. ## Introduction The Registry DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions: @@ -102,7 +102,7 @@ maps to the application user and so on. Wherever it is not easily distinguishabl âcontainerâ or suffix such as âapiâ. For example, an endpoint published as a management endpoint will be referenced with the name *management-api.griduser.yarncluster.com*. * Unique application name (per user) is not currently supported/guaranteed by YARN, but -it is supported by frameworks such as Apache Slider. The registry DNS service currently +it is supported by the YARN service framework. The registry DNS service currently leverages the last element of the ZK path entry for the application as an application name. These application names have to be unique for a given user. @@ -153,6 +153,7 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm | Name | Description | | ------------ | ------------- | +|hadoop.registry.zk.quorum| A comma separated list of hostname:port pairs defining the zookeeper quorum for the [YARN registry](../registry/registry-configuration.md). | | hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. | | hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. | | hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. | @@ -163,4 +164,37 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm | hadoop.registry.dns-ttl | The default TTL value to associate with DNS records. The default value is set to 1 (a value of 0 has undefined behavior). A typical value should be approximate to the time it takes YARN to restart a failed container. | | hadoop.registry.dns.zone-subnet | An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name. | | hadoop.registry.dns.zone-mask | The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name. | -| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.| \ No newline at end of file +| hadoop.registry.dns.zones-dir | A directory containing zone configuration files to read during zone initialization. This directory can contain zone master files named *zone-name.zone*. See [here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file documentation.| +### Sample configurations +``` + <property> + <description>The domain name for Hadoop cluster associated records.</description> + <name>hadoop.registry.dns.domain-name</name> + <value>ycluster</value> + </property> + + <property> + <description>The port number for the DNS listener. The default port is 5353. + If the standard privileged port 53 is used, make sure start the DNS with jsvc support.</description> + <name>hadoop.registry.dns.bind-port</name> + <value>5353</value> + </property> + + <property> + <description>The DNS functionality is enabled for the cluster. Default is false.</description> + <name>hadoop.registry.dns.enabled</name> + <value>true</value> + </property> + + <property> + <description>Address associated with the network interface to which the DNS listener should bind.</description> + <name>hadoop.registry.dns.bind-address</name> + <value>localhost</value> + </property> + + <property> + <description>A comma separated list of hostname:port pairs defining the zookeeper quorum for the YARN registry</description> + <name>hadoop.registry.zk.quorum</name> + <value>localhost:2181</value> + </property> + ``` \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md index a5dd0d2..ed56fa3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md @@ -73,6 +73,7 @@ assigned `0` since it is the first and only instance for the `hbasemaster` compo Below is the set of configurations in `yarn-site.xml` required for enabling Registry DNS. A full list of properties can be found in the Configuration section of [Registry DNS](RegistryDNS.md). + ``` <property> <description>The domain name for Hadoop cluster associated records.</description> @@ -84,7 +85,7 @@ section of [Registry DNS](RegistryDNS.md). <description>The port number for the DNS listener. The default port is 5353. If the standard privileged port 53 is used, make sure start the DNS with jsvc support.</description> <name>hadoop.registry.dns.bind-port</name> - <value>53</value> + <value>5353</value> </property> <property> @@ -94,6 +95,20 @@ section of [Registry DNS](RegistryDNS.md). </property> <property> + <description>Address associated with the network interface to which the DNS listener should bind.</description> + <name>hadoop.registry.dns.bind-address</name> + <value>localhost</value> + </property> + + <property> + <description> A comma separated list of hostname:port pairs defining the zookeeper quorum for the YARN registry</description> + <name>hadoop.registry.zk.quorum</name> + <value>localhost:2181</value> + </property> +``` +To configure Registry DNS to serve reverse lookup for `172.17.0.0/24` +``` + <property> <description>The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name.</description> <name>hadoop.registry.dns.zone-mask</name> @@ -104,11 +119,9 @@ section of [Registry DNS](RegistryDNS.md). <description>An indicator of the IP range associated with the cluster containers. The setting is utilized for the generation of the reverse zone name.</description> <name>hadoop.registry.dns.zone-subnet</name> - <value>172.17.0</value> + <value>172.17.0.0</value> </property> - ``` - ## Start the DNS Server By default, the DNS server runs on non-privileged port `5353`. Start the server with: --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
