Author: vinodkv
Date: Sat Aug 24 02:49:13 2013
New Revision: 1517103
URL: http://svn.apache.org/r1517103
Log:
YARN-1085. Modified YARN and MR2 web-apps to do HTTP authentication in secure
setup with kerberos. Contributed by Omkar Vinit Joshi.
svn merge --ignore-ancestry -c 1517101 ../../trunk/
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1517103&r1=1517102&r2=1517103&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Sat
Aug 24 02:49:13 2013
@@ -52,6 +52,9 @@ Release 2.1.1-beta - UNRELEASED
YARN-1082. Create base directories on HDFS after RM login to ensure RM
recovery doesn't fail in secure mode. (vinodkv via acmurthy)
+ YARN-1085. Modified YARN and MR2 web-apps to do HTTP authentication in
+ secure setup with kerberos. (Omkar Vinit Joshi via vinodkv)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1517103&r1=1517102&r2=1517103&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Sat Aug 24 02:49:13 2013
@@ -205,6 +205,12 @@ public class YarnConfiguration extends C
public static final String RM_KEYTAB =
RM_PREFIX + "keytab";
+ public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY =
+ RM_PREFIX + "webapp.spnego-principal";
+
+ public static final String RM_WEBAPP_SPENGO_KEYTAB_FILE_KEY =
+ RM_PREFIX + "webapp.spengo-keytab-file";
+
/** How long to wait until a container is considered dead.*/
public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS =
RM_PREFIX + "rm.container-allocation.expiry-interval-ms";
@@ -599,7 +605,13 @@ public class YarnConfiguration extends C
public static final String NM_USER_HOME_DIR =
NM_PREFIX + "user-home-dir";
-
+
+ public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY =
+ NM_PREFIX + "webapp.spnego-principal";
+
+ public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
+ NM_PREFIX + "webapp.spnego-keytab-file";
+
public static final String DEFAULT_NM_USER_HOME_DIR= "/home/";
////////////////////////////////
@@ -729,6 +741,12 @@ public class YarnConfiguration extends C
// Other Configs
////////////////////////////////
+ public static final String JHS_WEBAPP_SPNEGO_USER_NAME_KEY =
+ "jobhistoryserver.webapp.spnego-principal";
+
+ public static final String JHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
+ "jobhistoryserver.webapp.spnego-keytab-file";
+
/**
* The interval of the yarn client's querying application state after
* application submission. The unit is millisecond.
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1517103&r1=1517102&r2=1517103&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
Sat Aug 24 02:49:13 2013
@@ -33,6 +33,8 @@ import org.apache.commons.lang.StringUti
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,7 +67,6 @@ import com.google.inject.servlet.GuiceFi
@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
public class WebApps {
static final Logger LOG = LoggerFactory.getLogger(WebApps.class);
-
public static class Builder<T> {
static class ServletStruct {
public Class<? extends HttpServlet> clazz;
@@ -82,6 +83,8 @@ public class WebApps {
boolean findPort = false;
Configuration conf;
boolean devMode = false;
+ private String spnegoPrincipalKey;
+ private String spnegoKeytabKey;
private final HashSet<ServletStruct> servlets = new
HashSet<ServletStruct>();
private final HashMap<String, Object> attributes = new HashMap<String,
Object>();
@@ -135,6 +138,16 @@ public class WebApps {
this.conf = conf;
return this;
}
+
+ public Builder<T> withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) {
+ this.spnegoPrincipalKey = spnegoPrincipalKey;
+ return this;
+ }
+
+ public Builder<T> withHttpSpnegoKeytabKey(String spnegoKeytabKey) {
+ this.spnegoKeytabKey = spnegoKeytabKey;
+ return this;
+ }
public Builder<T> inDevMode() {
devMode = true;
@@ -197,8 +210,30 @@ public class WebApps {
}
}
HttpServer server =
- new HttpServer(name, bindAddress, port, findPort, conf,
- new AdminACLsManager(conf).getAdminAcl(), null,
webapp.getServePathSpecs());
+ new HttpServer(name, bindAddress, port, findPort, conf,
+ new AdminACLsManager(conf).getAdminAcl(), null,
+ webapp.getServePathSpecs()) {
+
+ {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ boolean initSpnego = true;
+ if (spnegoPrincipalKey == null ||
spnegoPrincipalKey.isEmpty()) {
+ LOG.warn("Principal for spnego filter is not set");
+ initSpnego = false;
+ }
+ if (spnegoKeytabKey == null || spnegoKeytabKey.isEmpty()) {
+ LOG.warn("Keytab for spnego filter is not set");
+ initSpnego = false;
+ }
+ if (initSpnego) {
+ LOG.info("Initializing spnego filter with principal key : "
+ + spnegoPrincipalKey + " keytab key : "
+ + spnegoKeytabKey);
+ initSpnego(conf, spnegoPrincipalKey, spnegoKeytabKey);
+ }
+ }
+ }
+ };
for(ServletStruct struct: servlets) {
server.addServlet(struct.name, struct.spec, struct.clazz);
}
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java?rev=1517103&r1=1517102&r2=1517103&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
Sat Aug 24 02:49:13 2013
@@ -59,8 +59,15 @@ public class WebServer extends AbstractS
LOG.info("Instantiating NMWebApp at " + bindAddress);
try {
this.webApp =
- WebApps.$for("node", Context.class, this.nmContext, "ws")
- .at(bindAddress).with(getConfig()).start(this.nmWebApp);
+ WebApps
+ .$for("node", Context.class, this.nmContext, "ws")
+ .at(bindAddress)
+ .with(getConfig())
+ .withHttpSpnegoPrincipalKey(
+ YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
+ .withHttpSpnegoKeytabKey(
+ YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+ .start(this.nmWebApp);
this.port = this.webApp.httpServer().getPort();
} catch (Exception e) {
String msg = "NMWebapps failed to start.";
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1517103&r1=1517102&r2=1517103&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Sat Aug 24 02:49:13 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
@@ -573,9 +574,16 @@ public class ResourceManager extends Com
protected void startWepApp() {
Builder<ApplicationMasterService> builder =
- WebApps.$for("cluster", ApplicationMasterService.class, masterService,
"ws").at(
- this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS));
+ WebApps
+ .$for("cluster", ApplicationMasterService.class, masterService,
+ "ws")
+ .with(conf)
+ .withHttpSpnegoPrincipalKey(
+ YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY)
+ .withHttpSpnegoKeytabKey(
+ YarnConfiguration.RM_WEBAPP_SPENGO_KEYTAB_FILE_KEY)
+ .at(this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS));
String proxyHostAndPort = YarnConfiguration.getProxyHostAndPort(conf);
if(YarnConfiguration.getRMWebAppHostAndPort(conf).
equals(proxyHostAndPort)) {
