YARN-2513. Host framework UIs in YARN for use with the ATS (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25f8f801 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25f8f801 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25f8f801 Branch: refs/heads/HADOOP-11890 Commit: 25f8f801d15e3d9f27f4a2a198262407203e14a5 Parents: d759b4b Author: Jonathan Eagles <jeag...@yahoo-inc.com> Authored: Wed Oct 21 15:38:08 2015 -0500 Committer: Jonathan Eagles <jeag...@yahoo-inc.com> Committed: Wed Oct 21 15:38:08 2015 -0500 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 2 + .../hadoop/yarn/conf/YarnConfiguration.java | 17 +++++++ .../org/apache/hadoop/yarn/webapp/WebApps.java | 18 +++++-- .../src/main/resources/yarn-default.xml | 6 +++ .../ApplicationHistoryServer.java | 44 ++++++++++++++-- .../TestApplicationHistoryServer.java | 53 +++++++++++++++++++- .../src/site/markdown/TimelineServer.md | 8 +++ 7 files changed, 140 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8277e05..ae26386 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -989,6 +989,8 @@ Release 2.7.2 - UNRELEASED YARN-2801. Add documentation for node labels feature. (Wangda Tan and Naganarasimha G R via ozawa) + YARN-2513. Host framework UIs in YARN for use with the ATS (jeagles) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 55b829f..3e89259 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1524,6 +1524,23 @@ public class YarnConfiguration extends Configuration { public static final String TIMELINE_SERVICE_PREFIX = YARN_PREFIX + "timeline-service."; + /** + * Comma seperated list of names for UIs hosted in the timeline server + * (For pluggable UIs). + */ + public static final String TIMELINE_SERVICE_UI_NAMES = + TIMELINE_SERVICE_PREFIX + "ui-names"; + + /** Relative web path that will serve up this UI (For pluggable UIs). */ + public static final String TIMELINE_SERVICE_UI_WEB_PATH_PREFIX = + TIMELINE_SERVICE_PREFIX + "ui-web-path."; + + /** + * Path to war file or static content directory for this UI + * (For pluggable UIs). + */ + public static final String TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX = + TIMELINE_SERVICE_PREFIX + "ui-on-disk-path."; // mark app-history related configs @Private as application history is going // to be integrated into the timeline service http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index bda24aa..0c6edad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -166,7 +166,7 @@ public class WebApps { return this; } - public WebApp start(WebApp webapp) { + public WebApp build(WebApp webapp) { if (webapp == null) { webapp = new WebApp() { @Override @@ -271,8 +271,7 @@ public class WebApps { webapp.setConf(conf); webapp.setHttpServer(server); - server.start(); - LOG.info("Web app /"+ name +" started at "+ server.getConnectorAddress(0).getPort()); + } catch (ClassNotFoundException e) { throw new WebAppException("Error starting http server", e); } catch (IOException e) { @@ -300,6 +299,19 @@ public class WebApps { return start(null); } + public WebApp start(WebApp webapp) { + WebApp webApp = build(webapp); + HttpServer2 httpServer = webApp.httpServer(); + try { + httpServer.start(); + LOG.info("Web app " + name + " started at " + + httpServer.getConnectorAddress(0).getPort()); + } catch (IOException e) { + throw new WebAppException("Error starting http server", e); + } + return webApp; + } + private String inferHostClass() { String thisClass = this.getClass().getName(); Throwable t = new Throwable(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 383e002..5dc4590 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1869,6 +1869,12 @@ </property> <property> + <description>Comma separated list of UIs that will be hosted</description> + <name>yarn.timeline-service.ui-names</name> + <value></value> + </property> + + <property> <description> Default maximum number of retries for timeline service client and value -1 means no limit. http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index dd571f8..dfacaf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.AuthenticationFilterInitializer; @@ -54,6 +55,8 @@ import org.apache.hadoop.yarn.server.timeline.webapp.CrossOriginFilterInitialize import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.mortbay.jetty.servlet.FilterHolder; +import org.mortbay.jetty.webapp.WebAppContext; import com.google.common.annotations.VisibleForTesting; @@ -137,6 +140,12 @@ public class ApplicationHistoryServer extends CompositeService { return this.ahsClientService; } + @Private + @VisibleForTesting + int getPort() { + return this.webApp.httpServer().getConnectorAddress(0).getPort(); + } + /** * @return ApplicationTimelineStore */ @@ -223,6 +232,7 @@ public class ApplicationHistoryServer extends CompositeService { timelineStore, new TimelineACLsManager(conf)); } + @SuppressWarnings("unchecked") private void startWebApp() { Configuration conf = getConfig(); TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager( @@ -278,14 +288,42 @@ public class ApplicationHistoryServer extends CompositeService { String bindAddress = WebAppUtils.getWebAppBindURL(conf, YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, WebAppUtils.getAHSWebAppURLWithoutScheme(conf)); - LOG.info("Instantiating AHSWebApp at " + bindAddress); try { + AHSWebApp ahsWebApp = new AHSWebApp(timelineDataManager, ahsClientService); webApp = WebApps .$for("applicationhistory", ApplicationHistoryClientService.class, ahsClientService, "ws") - .with(conf).at(bindAddress).start( - new AHSWebApp(timelineDataManager, ahsClientService)); + .with(conf).withAttribute(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS)).at(bindAddress).build(ahsWebApp); + HttpServer2 httpServer = webApp.httpServer(); + + String[] names = conf.getTrimmedStrings(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES); + WebAppContext webAppContext = httpServer.getWebAppContext(); + + for (String name : names) { + String webPath = conf.get( + YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + name); + String onDiskPath = conf.get( + YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX + name); + WebAppContext uiWebAppContext = new WebAppContext(); + uiWebAppContext.setContextPath(webPath); + uiWebAppContext.setWar(onDiskPath); + final String[] ALL_URLS = { "/*" }; + FilterHolder[] filterHolders = + webAppContext.getServletHandler().getFilters(); + for (FilterHolder filterHolder: filterHolders) { + if (!"guice".equals(filterHolder.getName())) { + HttpServer2.defineFilter(uiWebAppContext, filterHolder.getName(), + filterHolder.getClassName(), filterHolder.getInitParameters(), + ALL_URLS); + } + } + LOG.info("Hosting " + name + " from " + onDiskPath + " at " + webPath); + httpServer.addContext(uiWebAppContext, true); + } + httpServer.start(); + LOG.info("Instantiating AHSWebApp at " + getPort()); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index 01c309c..8dbccaa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -28,16 +28,20 @@ import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore; import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore; import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; -import org.junit.After; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; import java.util.HashMap; import java.util.Map; @@ -173,4 +177,49 @@ public class TestApplicationHistoryServer { } } + @Test(timeout = 240000) + public void testHostedUIs() throws Exception { + + ApplicationHistoryServer historyServer = new ApplicationHistoryServer(); + Configuration config = new YarnConfiguration(); + config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, + MemoryTimelineStore.class, TimelineStore.class); + config.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS, + MemoryTimelineStateStore.class, TimelineStateStore.class); + config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + "localhost:0"); + final String UI1 = "UI1"; + String connFileStr = ""; + + File diskFile = new File("./pom.xml"); + String diskFileStr = readInputStream(new FileInputStream(diskFile)); + try { + config.set(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES, UI1); + config.set(YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + UI1, + "/" + UI1); + config.set(YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX + + UI1, "./"); + historyServer.init(config); + historyServer.start(); + URL url = new URL("http://localhost:"; + historyServer.getPort() + "/" + + UI1 + "/pom.xml"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + connFileStr = readInputStream(conn.getInputStream()); + } finally { + historyServer.stop(); + } + assertEquals("Web file contents should be the same as on disk contents", + diskFileStr, connFileStr); + } + private String readInputStream(InputStream input) throws Exception { + ByteArrayOutputStream data = new ByteArrayOutputStream(); + byte[] buffer = new byte[512]; + int read; + while ((read = input.read(buffer)) >= 0) { + data.write(buffer, 0, read); + } + return new String(data.toByteArray(), "UTF-8"); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md index 1f0388e..2048012 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md @@ -185,6 +185,14 @@ selected if this policy is either of `HTTPS_ONLY` or `HTTP_AND_HTTPS`. | `yarn.timeline-service.client.retry-interval-ms` | The interval in milliseconds between retries for the timeline service client. Defaults to `1000`. | | `yarn.timeline-service.generic-application-history.max-applications` | The max number of applications could be fetched by using REST API or application history protocol and shown in timeline server web ui. Defaults to `10000`. | +#### UI Hosting Configuration + +The timeline service can host multiple UIs if enabled. The service can support both static web sites hosted in a directory or war files bundled. The web UI is then hosted on the timeline service HTTP port under the path configured. +| Configuration Property | Description | +|:---- |:---- | +| `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be hosted. Defaults to `none`. | +| `yarn.timeline-service.ui-on-disk-path.$name` | For each of the ui-names, an on disk path should be specified to the directory service static content or the location of a web archive (war file). | +| `yarn.timeline-service.ui-web-path.$name` | For each of the ui-names, the web path should be specified relative to the Timeline server root. Paths should begin with a starting slash. | #### Security Configuration
