MAPREDUCE-6632. Master.getMasterAddress() should be updated to use YARN-4629 (templedf via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4fc632ae Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4fc632ae Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4fc632ae Branch: refs/heads/MAPREDUCE-6608 Commit: 4fc632ae19a1d6b0ec09cc7ead789a3cab1c2f1c Parents: 40acace Author: Robert Kanter <rkan...@apache.org> Authored: Thu Sep 22 16:12:56 2016 -0700 Committer: Robert Kanter <rkan...@apache.org> Committed: Thu Sep 22 16:12:56 2016 -0700 ---------------------------------------------------------------------- .../hadoop-mapreduce-client-core/pom.xml | 6 +- .../java/org/apache/hadoop/mapred/Master.java | 70 ++++++-------------- .../org/apache/hadoop/mapred/TestMaster.java | 56 +--------------- 3 files changed, 28 insertions(+), 104 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fc632ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index 4de24c7..00bb11b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -37,7 +37,11 @@ <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-common</artifactId> + <artifactId>hadoop-yarn-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fc632ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java index d84e395..ec00405 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java @@ -19,75 +19,45 @@ package org.apache.hadoop.mapred; import java.io.IOException; -import java.net.InetSocketAddress; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.yarn.conf.HAUtil; -import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.client.util.YarnClientUtils; @Private @Unstable public class Master { - - private static final Log LOG = LogFactory.getLog(Master.class); - public enum State { INITIALIZING, RUNNING; } - public static String getMasterUserName(Configuration conf) { - String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) { - return conf.get(MRConfig.MASTER_USER_NAME); - } - else { - return conf.get(YarnConfiguration.RM_PRINCIPAL); - } + public static String getMasterAddress(Configuration conf) { + String masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012"); + + return NetUtils.createSocketAddr(masterAddress, 8012, + MRConfig.MASTER_ADDRESS).getHostName(); } - - public static InetSocketAddress getMasterAddress(Configuration conf) { - String masterAddress; - String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); + + public static String getMasterPrincipal(Configuration conf) + throws IOException { + String masterPrincipal; + String framework = conf.get(MRConfig.FRAMEWORK_NAME, + MRConfig.YARN_FRAMEWORK_NAME); + if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) { - masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012"); - return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS); - } else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) && - HAUtil.isHAEnabled(conf)) { - YarnConfiguration yarnConf = new YarnConfiguration(conf); - if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) { - String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS); - if (rmIds != null && rmIds.length > 0) { - // If RM_HA_ID is not configured, use the first one. - // Because any valid RM HA ID should work. - yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]); - } else { - LOG.warn("RM_HA_IDS is not configured when RM HA is enabled"); - } - } - return yarnConf.getSocketAddr( - YarnConfiguration.RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_PORT); + String masterAddress = getMasterAddress(conf); + // get kerberos principal for use as delegation token renewer + masterPrincipal = + SecurityUtil.getServerPrincipal(conf.get(MRConfig.MASTER_USER_NAME), + masterAddress); } else { - return conf.getSocketAddr( - YarnConfiguration.RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_PORT); + masterPrincipal = YarnClientUtils.getRmPrincipal(conf); } - } - public static String getMasterPrincipal(Configuration conf) - throws IOException { - String masterHostname = getMasterAddress(conf).getHostName(); - // get kerberos principal for use as delegation token renewer - return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname); + return masterPrincipal; } - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fc632ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java index 498abbc..0f4ebee 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java @@ -18,13 +18,10 @@ package org.apache.hadoop.mapred; -import static org.junit.Assert.*; - -import java.net.InetSocketAddress; - import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import org.junit.Test; public class TestMaster { @@ -33,13 +30,6 @@ public class TestMaster { public void testGetMasterAddress() { YarnConfiguration conf = new YarnConfiguration(); - // Default is yarn framework - String masterHostname = Master.getMasterAddress(conf).getHostName(); - - // no address set so should default to default rm address - InetSocketAddress rmAddr = NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS); - assertEquals(masterHostname, rmAddr.getHostName()); - // Trying invalid master address for classic conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); conf.set(MRConfig.MASTER_ADDRESS, "local:invalid"); @@ -55,47 +45,7 @@ public class TestMaster { // Change master address to a valid value conf.set(MRConfig.MASTER_ADDRESS, "bar.com:8042"); - masterHostname = Master.getMasterAddress(conf).getHostName(); + String masterHostname = Master.getMasterAddress(conf); assertEquals(masterHostname, "bar.com"); - - // change framework to yarn - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192"); - masterHostname = Master.getMasterAddress(conf).getHostName(); - assertEquals(masterHostname, "foo1.com"); - - // change framework to yarn and enable HA - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); - conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2"); - conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192"); - conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192"); - masterHostname = Master.getMasterAddress(conf).getHostName(); - // If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used. - assertEquals(masterHostname, "rm1.com"); - conf.set(YarnConfiguration.RM_HA_ID, "rm2"); - masterHostname = Master.getMasterAddress(conf).getHostName(); - // If RM_HA_ID is configured, use the given RM_HA_ID. - assertEquals(masterHostname, "rm2.com"); } - - @Test - public void testGetMasterUser() { - YarnConfiguration conf = new YarnConfiguration(); - conf.set(MRConfig.MASTER_USER_NAME, "foo"); - conf.set(YarnConfiguration.RM_PRINCIPAL, "bar"); - - // default is yarn framework - assertEquals(Master.getMasterUserName(conf), "bar"); - - // set framework name to classic - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); - assertEquals(Master.getMasterUserName(conf), "foo"); - - // change framework to yarn - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - assertEquals(Master.getMasterUserName(conf), "bar"); - - } - } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
