Author: vinodkv Date: Tue Sep 24 17:20:48 2013 New Revision: 1525948 URL: http://svn.apache.org/r1525948 Log: YARN-1204. Added separate configuration properties for https for RM and NM without which servers enabled with https will also start on http ports. Contributed by Omkar Vinit Joshi. MAPREDUCE-5523. Added separate configuration properties for https for JHS without which even when https is enabled, it starts on http port itself. Contributed by Omkar Vinit Joshi. svn merge --ignore-ancestry -c 1525947 ../../trunk/
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1525948&r1=1525947&r2=1525948&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Sep 24 17:20:48 2013 @@ -84,6 +84,10 @@ Release 2.1.1-beta - 2013-09-23 MAPREDUCE-5379. Include token tracking ids in jobconf. (kkambatl via tucu) + MAPREDUCE-5523. Added separate configuration properties for https for JHS + without which even when https is enabled, it starts on http port itself. + (Omkar Vinit Joshi via vinodkv) + OPTIMIZATIONS MAPREDUCE-5446. TestJobHistoryEvents and TestJobHistoryParsing have race Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java?rev=1525948&r1=1525947&r2=1525948&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java Tue Sep 24 17:20:48 2013 @@ -37,11 +37,11 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AppInfo; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.Controller; import org.apache.hadoop.yarn.webapp.View; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.base.Joiner; import com.google.inject.Inject; @@ -62,7 +62,7 @@ public class AppController extends Contr set(APP_ID, app.context.getApplicationID().toString()); set(RM_WEB, JOINER.join(WebAppUtil.getSchemePrefix(), - YarnConfiguration.getRMWebAppHostAndPort(conf))); + WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf))); } @Inject Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java?rev=1525948&r1=1525947&r2=1525948&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java Tue Sep 24 17:20:48 2013 @@ -17,6 +17,10 @@ */ package org.apache.hadoop.mapreduce.v2.app.webapp; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; + public class WebAppUtil { private static boolean isSSLEnabledInYARN; @@ -36,4 +40,21 @@ public class WebAppUtil { return "http://"; } } -} + + public static void setJHSWebAppURLWithoutScheme(Configuration conf, + String hostAddress) { + if (HttpConfig.isSecure()) { + conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS, hostAddress); + } else { + conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, hostAddress); + } + } + + public static String getJHSWebAppURLWithoutScheme(Configuration conf) { + if (HttpConfig.isSecure()) { + return conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS); + } else { + return conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS); + } + } +} \ No newline at end of file Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java?rev=1525948&r1=1525947&r2=1525948&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java Tue Sep 24 17:20:48 2013 @@ -18,8 +18,15 @@ package org.apache.hadoop.mapreduce.v2.jobhistory; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.net.NetUtils; /** * Stores Job History configuration keys that can be set by administrators of @@ -130,6 +137,13 @@ public class JHAdminConfig { public static final String DEFAULT_MR_HISTORY_WEBAPP_ADDRESS = "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_PORT; + /**The https address the history server webapp is on.*/ + public static final String MR_HISTORY_WEBAPP_HTTPS_ADDRESS = + MR_HISTORY_PREFIX + "webapp.https.address"; + public static final int DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT = 19890; + public static final String DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS = + "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT; + /**The kerberos principal to be used for spnego filter for history server*/ public static final String MR_WEBAPP_SPNEGO_USER_NAME_KEY = MR_HISTORY_PREFIX + "webapp.spnego-principal"; @@ -160,4 +174,36 @@ public class JHAdminConfig { */ public static boolean DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS = false; + public static String getResolvedMRHistoryWebAppURLWithoutScheme( + Configuration conf) { + InetSocketAddress address = null; + if (HttpConfig.isSecure()) { + address = + conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT); + } else { + address = + conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT); } + address = NetUtils.getConnectAddress(address); + StringBuffer sb = new StringBuffer(); + InetAddress resolved = address.getAddress(); + if (resolved == null || resolved.isAnyLocalAddress() || + resolved.isLoopbackAddress()) { + String lh = address.getHostName(); + try { + lh = InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + //Ignore and fallback. + } + sb.append(lh); + } else { + sb.append(address.getHostName()); + } + sb.append(":").append(address.getPort()); + return sb.toString(); + } + } Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1525948&r1=1525947&r2=1525948&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Tue Sep 24 17:20:48 2013 @@ -28,10 +28,12 @@ import org.apache.hadoop.fs.CommonConfig import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapred.LocalContainerLauncher; import org.apache.hadoop.mapred.ShuffleHandler; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.v2.app.webapp.WebAppUtil; import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; @@ -43,6 +45,8 @@ import org.apache.hadoop.yarn.exceptions import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.hamcrest.core.IsEqual; /** * Configures and starts the MR-specific components in the YARN cluster. @@ -155,8 +159,7 @@ public class MiniMRYarnCluster extends M // pick free random ports. getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, hostname + ":0"); - getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, - hostname + ":0"); + WebAppUtil.setJHSWebAppURLWithoutScheme(getConfig(), hostname + ":0"); getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS, hostname + ":0"); } @@ -182,17 +185,17 @@ public class MiniMRYarnCluster extends M //need to do this because historyServer.init creates a new Configuration getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); - getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, - historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)); + WebAppUtil.setJHSWebAppURLWithoutScheme(getConfig(), + WebAppUtil.getJHSWebAppURLWithoutScheme(historyServer.getConfig())); LOG.info("MiniMRYARN ResourceManager address: " + getConfig().get(YarnConfiguration.RM_ADDRESS)); LOG.info("MiniMRYARN ResourceManager web address: " + - getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS)); + WebAppUtils.getRMWebAppURLWithoutScheme(getConfig())); LOG.info("MiniMRYARN HistoryServer address: " + getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); LOG.info("MiniMRYARN HistoryServer web address: " + - getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)); + JHAdminConfig.getResolvedMRHistoryWebAppURLWithoutScheme(getConfig())); } @Override