Author: vinodkv
Date: Thu Sep 19 23:35:58 2013
New Revision: 1524864
URL: http://svn.apache.org/r1524864
Log:
YARN-1203. Changed YARN web-app proxy to handle http and https URLs from AM
registration and finish correctly. Contributed by Omkar Vinit Joshi.
MAPREDUCE-5515. Fixed MR AM's webapp to depend on a new config
mapreduce.ssl.enabled to enable https and disabling it by default as MR AM needs
to set up its own certificates etc and not depend on clusters'. Contributed by
Omkar Vinit Joshi.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1524864&r1=1524863&r2=1524864&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Sep 19 23:35:58 2013
@@ -126,6 +126,9 @@ Release 2.1.1-beta - 2013-09-23
YARN-1001. Added a web-service to get statistics about per application-type
per state for consumption by downstream projects. (Zhijie Shen via vinodkv)
+ YARN-1203. Changed YARN web-app proxy to handle http and https URLs from
+ AM registration and finish correctly. (Omkar Vinit Joshi via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java?rev=1524864&r1=1524863&r2=1524864&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
Thu Sep 19 23:35:58 2013
@@ -91,6 +91,8 @@ public abstract class FinishApplicationM
/**
* Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ * This url if contains scheme then that will be used by resource manager
+ * web application proxy otherwise it will default to http.
* @return <em>tracking URL</em>for the <code>ApplicationMaster</code>
*/
@Public
@@ -99,6 +101,8 @@ public abstract class FinishApplicationM
/**
* Set the <em>tracking URL</em>for the <code>ApplicationMaster</code>
+ * This url if contains scheme then that will be used by resource manager
+ * web application proxy otherwise it will default to http.
* @param url <em>tracking URL</em>for the
* <code>ApplicationMaster</code>
*/
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java?rev=1524864&r1=1524863&r2=1524864&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
Thu Sep 19 23:35:58 2013
@@ -103,6 +103,8 @@ public abstract class RegisterApplicatio
/**
* Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ * This url if contains scheme then that will be used by resource manager
+ * web application proxy otherwise it will default to http.
* @return <em>tracking URL</em> for the <code>ApplicationMaster</code>
*/
@Public
@@ -111,6 +113,8 @@ public abstract class RegisterApplicatio
/**
* Set the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ * This url if contains scheme then that will be used by resource manager
+ * web application proxy otherwise it will default to http.
* @param trackingUrl <em>tracking URL</em> for the
* <code>ApplicationMaster</code>
*/
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1524864&r1=1524863&r2=1524864&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Thu Sep 19 23:35:58 2013
@@ -26,6 +26,8 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -865,7 +867,8 @@ public class YarnConfiguration extends C
}
public static String getRMWebAppURL(Configuration conf) {
- return JOINER.join("http://";, getRMWebAppHostAndPort(conf));
+ return JOINER.join(HttpConfig.getSchemePrefix(),
+ getRMWebAppHostAndPort(conf));
}
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java?rev=1524864&r1=1524863&r2=1524864&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
Thu Sep 19 23:35:58 2013
@@ -137,14 +137,44 @@ public class ProxyUriUtils {
/**
* Create a URI form a no scheme Url, such as is returned by the AM.
- * @param noSchemeUrl the URL formate returned by an AM
+ * @param url the URL format returned by an AM. This may or may not contain
+ * scheme.
* @return a URI with an http scheme
* @throws URISyntaxException if the url is not formatted correctly.
*/
- public static URI getUriFromAMUrl(String noSchemeUrl)
+ public static URI getUriFromAMUrl(String url)
throws URISyntaxException {
- return new URI(HttpConfig.getSchemePrefix() + noSchemeUrl);
+ if (getSchemeFromUrl(url).isEmpty()) {
+ /*
+ * check is made to make sure if AM reports with scheme then it will be
+ * used by default otherwise it will default to the one configured using
+ * "hadoop.ssl.enabled".
+ */
+ return new URI(HttpConfig.getSchemePrefix() + url);
+ } else {
+ return new URI(url);
+ }
}
+
+ /**
+ * Create a URI form a no scheme Url, such as is returned by the AM.
+ * @param noSchemeUrl the URL formate returned by an AM
+ * @return a URI with an http scheme
+ * @throws URISyntaxException if the url is not formatted correctly.
+ */
+ public static URI getUriFromAMUrl(String scheme, String noSchemeUrl)
+ throws URISyntaxException {
+ if (getSchemeFromUrl(noSchemeUrl).isEmpty()) {
+ /*
+ * check is made to make sure if AM reports with scheme then it will be
+ * used by default otherwise it will default to the one configured
using
+ * "hadoop.ssl.enabled".
+ */
+ return new URI(scheme + "://" + noSchemeUrl);
+ } else {
+ return new URI(noSchemeUrl);
+ }
+ }
/**
* Returns the first valid tracking link, if any, from the given id from the
@@ -169,4 +199,20 @@ public class ProxyUriUtils {
}
return null;
}
+
+ /**
+ * Returns the scheme if present in the url
+ * eg. "https://issues.apache.org/jira/browse/YARN"; > "https"
+ */
+ public static String getSchemeFromUrl(String url) {
+ int index = 0;
+ if (url != null) {
+ index = url.indexOf("://");
+ }
+ if (index > 0) {
+ return url.substring(0, index);
+ } else {
+ return "";
+ }
+ }
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java?rev=1524864&r1=1524863&r2=1524864&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
Thu Sep 19 23:35:58 2013
@@ -163,7 +163,6 @@ public class WebAppProxyServlet extends
}
config.setLocalAddress(localAddress);
HttpMethod method = new GetMethod(uri.getEscapedURI());
-
@SuppressWarnings("unchecked")
Enumeration<String> names = req.getHeaderNames();
while(names.hasMoreElements()) {
@@ -293,14 +292,17 @@ public class WebAppProxyServlet extends
}
String original = applicationReport.getOriginalTrackingUrl();
URI trackingUri = null;
- if (original != null) {
- trackingUri = ProxyUriUtils.getUriFromAMUrl(original);
- }
// fallback to ResourceManager's app page if no tracking URI provided
if(original == null || original.equals("N/A")) {
resp.sendRedirect(resp.encodeRedirectURL(
StringHelper.pjoin(rmAppPageUrlBase, id.toString())));
return;
+ } else {
+ if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) {
+ trackingUri = ProxyUriUtils.getUriFromAMUrl("http", original);
+ } else {
+ trackingUri = new URI(original);
+ }
}
String runningUser = applicationReport.getUser();
@@ -311,8 +313,7 @@ public class WebAppProxyServlet extends
req.getQueryString(), true), runningUser, id);
return;
}
-
- URI toFetch = new URI(req.getScheme(),
+ URI toFetch = new URI(trackingUri.getScheme(),
trackingUri.getAuthority(),
StringHelper.ujoin(trackingUri.getPath(), rest),
req.getQueryString(),
null);
