Repository: ambari Updated Branches: refs/heads/trunk f715c3a4f -> f58782c74
AMBARI-5863. Slider Apps view API for app does not populate subresources. (srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f58782c7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f58782c7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f58782c7 Branch: refs/heads/trunk Commit: f58782c7405930d6629f6f819d84d64d32d92a97 Parents: f715c3a Author: Srimanth Gunturi <sgunt...@hortonworks.com> Authored: Thu May 22 18:36:36 2014 -0700 Committer: Srimanth Gunturi <sgunt...@hortonworks.com> Committed: Thu May 22 18:36:52 2014 -0700 ---------------------------------------------------------------------- .../slider/slider/0.30.0/slider-0.30.0.jar | Bin 1024630 -> 1029968 bytes .../view/slider/SliderAppsResourceProvider.java | 4 +- .../view/slider/SliderAppsViewController.java | 30 +++- .../slider/SliderAppsViewControllerImpl.java | 140 ++++++++++++++-- .../view/slider/rest/client/BaseHttpClient.java | 20 +++ .../rest/client/SliderAppMasterClient.java | 161 +++++++++++++++++++ 6 files changed, 340 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/lib/org/apache/slider/slider/0.30.0/slider-0.30.0.jar ---------------------------------------------------------------------- diff --git a/contrib/views/slider/lib/org/apache/slider/slider/0.30.0/slider-0.30.0.jar b/contrib/views/slider/lib/org/apache/slider/slider/0.30.0/slider-0.30.0.jar index 7c0ac2d..6e7b8d1 100644 Binary files a/contrib/views/slider/lib/org/apache/slider/slider/0.30.0/slider-0.30.0.jar and b/contrib/views/slider/lib/org/apache/slider/slider/0.30.0/slider-0.30.0.jar differ http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsResourceProvider.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsResourceProvider.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsResourceProvider.java index c5237fc..138a1ab 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsResourceProvider.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsResourceProvider.java @@ -62,7 +62,7 @@ public class SliderAppsResourceProvider implements ResourceProvider<SliderApp> { throws SystemException, NoSuchResourceException, UnsupportedPropertyException { try { - SliderApp sliderApp = sliderController.getSliderApp(resourceId); + SliderApp sliderApp = sliderController.getSliderApp(resourceId, properties); if (sliderApp == null) throw new NoSuchResourceException(resourceId); return sliderApp; @@ -81,7 +81,7 @@ public class SliderAppsResourceProvider implements ResourceProvider<SliderApp> { UnsupportedPropertyException { Set<SliderApp> appSet = new HashSet<SliderApp>(); try { - List<SliderApp> sliderApps = sliderController.getSliderApps(); + List<SliderApp> sliderApps = sliderController.getSliderApps(request.getPropertyIds()); for (SliderApp app : sliderApps) appSet.add(app); } catch (YarnException e) { http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewController.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewController.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewController.java index 3cc7ab8..90ff76c 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewController.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewController.java @@ -20,6 +20,7 @@ package org.apache.ambari.view.slider; import java.io.IOException; import java.util.List; +import java.util.Set; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -30,7 +31,30 @@ public interface SliderAppsViewController { public ViewStatus getViewStatus(); - public SliderApp getSliderApp(String applicationId) throws YarnException, IOException; - - public List<SliderApp> getSliderApps() throws YarnException, IOException; + /** + * Provides information about requested Slider App. + * + * @param applicationId + * @param properties + * Identifies specific properties to show up. Provide + * <code>null</code> for default properties. + * @return + * @throws YarnException + * @throws IOException + */ + public SliderApp getSliderApp(String applicationId, Set<String> properties) + throws YarnException, IOException; + + /** + * Provides list of Slider apps with requested properties populated. + * + * @param properties + * Identifies specific properties to show up. Provide + * <code>null</code> for default properties. + * @return + * @throws YarnException + * @throws IOException + */ + public List<SliderApp> getSliderApps(Set<String> properties) + throws YarnException, IOException; } http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java index 8c1da9c..6a884ce 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java @@ -20,8 +20,11 @@ package org.apache.ambari.view.slider; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; import org.apache.ambari.view.ViewContext; import org.apache.ambari.view.slider.clients.AmbariClient; @@ -30,6 +33,8 @@ import org.apache.ambari.view.slider.clients.AmbariClusterInfo; import org.apache.ambari.view.slider.clients.AmbariHostComponent; import org.apache.ambari.view.slider.clients.AmbariService; import org.apache.ambari.view.slider.clients.AmbariServiceInfo; +import org.apache.ambari.view.slider.rest.client.SliderAppMasterClient; +import org.apache.ambari.view.slider.rest.client.SliderAppMasterClient.SliderAppMasterData; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -37,7 +42,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.log4j.Logger; +import org.apache.slider.api.ClusterDescription; import org.apache.slider.client.SliderClient; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -137,8 +145,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } @Override - public SliderApp getSliderApp(String applicationId) throws YarnException, - IOException { + public SliderApp getSliderApp(String applicationId, Set<String> properties) + throws YarnException, IOException { if (applicationId != null) { int index = applicationId.indexOf('_'); if (index > -1 && index < applicationId.length() - 1) { @@ -150,9 +158,9 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); try { - ApplicationReport yarnApp = getSliderClient().getApplicationReport( - appId); - return mapToSliderApp(yarnApp); + SliderClient sliderClient = getSliderClient(); + ApplicationReport yarnApp = sliderClient.getApplicationReport(appId); + return populateSliderApp(yarnApp, properties, sliderClient); } finally { Thread.currentThread().setContextClassLoader(currentClassLoader); } @@ -161,7 +169,8 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { return null; } - private SliderApp mapToSliderApp(ApplicationReport yarnApp) { + private SliderApp populateSliderApp(ApplicationReport yarnApp, + Set<String> properties, SliderClient sliderClient) { if (yarnApp == null) return null; SliderApp app = new SliderApp(); @@ -175,6 +184,110 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { app.setType(yarnApp.getApplicationType()); app.setStartTime(yarnApp.getStartTime()); app.setEndTime(yarnApp.getFinishTime()); + if (properties != null && !properties.isEmpty()) { + SliderAppMasterClient sliderAppClient = yarnApp.getTrackingUrl() == null ? null + : new SliderAppMasterClient(yarnApp.getTrackingUrl()); + SliderAppMasterData appMasterData = null; + Map<String, String> quickLinks = new HashMap<String, String>(); + for (String property : properties) { + if ("RUNNING".equals(app.getState())) { + if (sliderAppClient != null) { + if (appMasterData == null) + appMasterData = sliderAppClient.getAppMasterData(); + if ("urls".equals(property.toLowerCase())) { + if (quickLinks.isEmpty()) + quickLinks = sliderAppClient + .getQuickLinks(appMasterData.publisherUrl); + app.setUrls(quickLinks); + } else if ("configs".equals(property.toLowerCase())) { + Map<String, Map<String, String>> configs = sliderAppClient + .getConfigs(appMasterData.publisherUrl); + app.setConfigs(configs); + } else if ("jmx".equals(property.toLowerCase())) { + if (quickLinks.isEmpty()) + quickLinks = sliderAppClient + .getQuickLinks(appMasterData.publisherUrl); + if (quickLinks != null && quickLinks.containsKey("JMX")) { + String jmxUrl = quickLinks.get("JMX"); + app.setJmx(sliderAppClient.getJmx(jmxUrl)); + } + Map<String, Map<String, String>> configs = sliderAppClient + .getConfigs(appMasterData.publisherUrl); + app.setConfigs(configs); + } else if ("components".equals(property.toLowerCase())) { + try { + System.setProperty(SliderKeys.HADOOP_USER_NAME, "yarn"); + ClusterDescription description = sliderClient + .getClusterDescription(yarnApp.getName()); + if (description != null && description.status != null + && !description.status.isEmpty()) { + Map<String, SliderAppComponent> componentTypeMap = new HashMap<String, SliderAppComponent>(); + for (Entry<String, Object> e : description.status.entrySet()) { + @SuppressWarnings("unchecked") + Map<String, Map<String, Map<String, Object>>> componentsObj = (Map<String, Map<String, Map<String, Object>>>) e + .getValue(); + boolean isLive = "live".equals(e.getKey()); + for (Entry<String, Map<String, Map<String, Object>>> componentEntry : componentsObj + .entrySet()) { + SliderAppComponent appComponent = componentTypeMap + .get(componentEntry.getKey()); + if (appComponent == null) { + appComponent = new SliderAppComponent(); + appComponent.setComponentName(componentEntry.getKey()); + appComponent + .setActiveContainers(new HashMap<String, Map<String, String>>()); + appComponent + .setCompletedContainers(new HashMap<String, Map<String, String>>()); + componentTypeMap.put(componentEntry.getKey(), + appComponent); + } + for (Entry<String, Map<String, Object>> containerEntry : componentEntry + .getValue().entrySet()) { + Map<String, String> containerDataMap = new HashMap<String, String>(); + String containerId = containerEntry.getKey(); + Map<String, Object> containerValues = containerEntry + .getValue(); + for (String containerProperty : containerValues + .keySet()) { + Object containerPropertyValue = containerValues + .get(containerProperty); + containerDataMap.put(containerProperty, + containerPropertyValue.toString()); + } + if (isLive) + appComponent.getActiveContainers().put(containerId, + containerDataMap); + else + appComponent.getCompletedContainers().put( + containerId, containerDataMap); + } + appComponent.setInstanceCount(appComponent + .getActiveContainers().size() + + appComponent.getCompletedContainers().size()); + } + } + app.setComponents(componentTypeMap); + } + } catch (UnknownApplicationInstanceException e) { + logger.warn( + "Unable to determine app components for " + + yarnApp.getName(), e); + } catch (YarnException e) { + logger.warn( + "Unable to determine app components for " + + yarnApp.getName(), e); + throw new RuntimeException(e.getMessage(), e); + } catch (IOException e) { + logger.warn( + "Unable to determine app components for " + + yarnApp.getName(), e); + throw new RuntimeException(e.getMessage(), e); + } + } + } + } + } + } return app; } @@ -188,7 +301,12 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { protected SliderClient getSliderClient() { Configuration sliderClientConfiguration = getSliderClientConfiguration(); if (sliderClientConfiguration != null) { - SliderClient client = new SliderClient(); + SliderClient client = new SliderClient() { + @Override + public String getUsername() throws IOException { + return null; + } + }; try { sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, new String[] { "usage" }); @@ -257,15 +375,17 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController { } @Override - public List<SliderApp> getSliderApps() throws YarnException, IOException { + public List<SliderApp> getSliderApps(Set<String> properties) + throws YarnException, IOException { List<SliderApp> sliderApps = new ArrayList<SliderApp>(); ClassLoader currentClassLoader = Thread.currentThread() .getContextClassLoader(); Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); try { - List<ApplicationReport> yarnApps = getSliderClient().getApplications(); + SliderClient sliderClient = getSliderClient(); + List<ApplicationReport> yarnApps = sliderClient.getApplications(); for (ApplicationReport yarnApp : yarnApps) { - sliderApps.add(mapToSliderApp(yarnApp)); + sliderApps.add(populateSliderApp(yarnApp, properties, sliderClient)); } } finally { Thread.currentThread().setContextClassLoader(currentClassLoader); http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/BaseHttpClient.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/BaseHttpClient.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/BaseHttpClient.java index 139494c..70778af 100644 --- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/BaseHttpClient.java +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/BaseHttpClient.java @@ -85,6 +85,11 @@ public class BaseHttpClient { } public JsonElement doGetJson(String path) throws HttpException, IOException { + return doGetJson(getUrl(), path); + } + + public JsonElement doGetJson(String url, String path) throws HttpException, + IOException { GetMethod get = new GetMethod(url + path); if (isNeedsAuthentication()) { get.setDoAuthentication(true); @@ -101,6 +106,21 @@ public class BaseHttpClient { return null; } + public String doGet(String path) throws HttpException, IOException { + GetMethod get = new GetMethod(url + path); + if (isNeedsAuthentication()) { + get.setDoAuthentication(true); + } + int executeMethod = getHttpClient().executeMethod(get); + switch (executeMethod) { + case HttpStatus.SC_OK: + return get.getResponseBodyAsString(); + default: + break; + } + return null; + } + private HttpClient getHttpClient() { if (httpClient == null) { httpClient = new HttpClient(); http://git-wip-us.apache.org/repos/asf/ambari/blob/f58782c7/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java ---------------------------------------------------------------------- diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java new file mode 100644 index 0000000..db29035 --- /dev/null +++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/rest/client/SliderAppMasterClient.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.slider.rest.client; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.httpclient.HttpException; +import org.apache.log4j.Logger; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +public class SliderAppMasterClient extends BaseHttpClient { + + private static final Logger logger = Logger + .getLogger(SliderAppMasterClient.class); + + public static class SliderAppMasterData { + public String registryUrl; + public String uiUrl; + public String managementUrl; + public String publisherUrl; + } + + public static class SliderAppContainerData { + public String hostName; + public String containerId; + } + + public SliderAppMasterClient(String url) { + super(url); + } + + public SliderAppMasterData getAppMasterData() { + try { + String html = doGet(""); + int from = html.lastIndexOf("<ul>"); + int to = html.lastIndexOf("</ul>"); + if (from < to && from > -1) { + SliderAppMasterData data = new SliderAppMasterData(); + String content = html.substring(from, to); + content = content.replaceAll("<[^>]*>", "\r\n"); + String[] splits = content.split("\r\n"); + for (int i = 0; i < splits.length; i++) { + String split = splits[i].trim(); + if ("Registry Web Service".equals(split)) + data.registryUrl = splits[i + 1].trim(); + else if ("Application Master Web UI".equals(split)) + data.uiUrl = splits[i + 1].trim(); + else if ("Management REST API".equals(split)) + data.managementUrl = splits[i + 1].trim(); + else if ("Publisher Service".equals(split)) + data.publisherUrl = splits[i + 1].trim(); + } + return data; + } + } catch (HttpException e) { + logger.warn("Unable to determine Ambari clusters", e); + throw new RuntimeException(e.getMessage(), e); + } catch (IOException e) { + logger.warn("Unable to determine Ambari clusters", e); + throw new RuntimeException(e.getMessage(), e); + } + return null; + } + + public Map<String, String> getQuickLinks(String providerUrl) { + try { + JsonElement json = super.doGetJson(providerUrl, "/slider/quicklinks"); + Map<String, String> quickLinks = new HashMap<String, String>(); + JsonObject jsonObject = json.getAsJsonObject().get("entries") + .getAsJsonObject(); + for (Entry<String, JsonElement> entry : jsonObject.entrySet()) { + if ("org.apache.slider.jmx".equals(entry.getKey())) { + quickLinks.put("JMX", entry.getValue().getAsString()); + } else if ("org.apache.slider.monitor".equals(entry.getKey())) { + quickLinks.put("UI", entry.getValue().getAsString()); + } else if ("org.apache.slider.metrics".equals(entry.getKey())) { + quickLinks.put("Metrics", entry.getValue().getAsString()); + } else { + quickLinks.put(entry.getKey(), entry.getValue().getAsString()); + } + } + return quickLinks; + } catch (HttpException e) { + logger.warn("Unable to determine quicklinks from " + providerUrl, e); + throw new RuntimeException(e.getMessage(), e); + } catch (IOException e) { + logger.warn("Unable to determine quicklinks from " + providerUrl, e); + throw new RuntimeException(e.getMessage(), e); + } + } + + public Map<String, Map<String, String>> getConfigs(String providerUrl) { + try { + Map<String, Map<String, String>> configsMap = new HashMap<String, Map<String, String>>(); + JsonElement json = super.doGetJson(providerUrl, "/slider"); + if (json != null) { + JsonObject configsJson = json.getAsJsonObject().get("configurations") + .getAsJsonObject(); + for (Entry<String, JsonElement> entry : configsJson.entrySet()) { + if ("complete-config".equals(entry.getKey()) + || "quicklinks".equals(entry.getKey())) + continue; + JsonElement entryJson = super.doGetJson(providerUrl, "/slider/" + + entry.getKey()); + if (entryJson != null) { + JsonObject configsObj = entryJson.getAsJsonObject().get("entries") + .getAsJsonObject(); + if (configsObj != null) { + Map<String, String> configs = new HashMap<String, String>(); + for (Entry<String, JsonElement> e : configsObj.entrySet()) { + configs.put(e.getKey(), e.getValue().getAsString()); + } + configsMap.put(entry.getKey(), configs); + } + } + } + } + return configsMap; + } catch (HttpException e) { + logger.warn("Unable to determine quicklinks from " + providerUrl, e); + throw new RuntimeException(e.getMessage(), e); + } catch (IOException e) { + logger.warn("Unable to determine quicklinks from " + providerUrl, e); + throw new RuntimeException(e.getMessage(), e); + } + } + + /** + * Provides only the interesting JMX metric names and values. + * + * @param jmxUrl + * @return + */ + public Map<String, String> getJmx(String jmxUrl) { + // TODO Mechanism to filter JMX beans to get only interesting metrics is + // needed. + return null; + } + +}
