Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-12556 e3d5344a6 -> 13d806b85
AMBARI-20777 : AMS changes to use instanceId for cluster based segregation of data. (Addendum patch) (avijayan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8a731458 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8a731458 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8a731458 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 8a73145839f4b34c2231faac04ad10e64c096a87 Parents: 6a0b2a0 Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Wed Apr 26 11:15:02 2017 -0700 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Wed Apr 26 11:15:02 2017 -0700 ---------------------------------------------------------------------- .../timeline/HBaseTimelineMetricStore.java | 48 +++++++++++++++++++- .../metrics/timeline/PhoenixHBaseAccessor.java | 4 +- .../metrics/timeline/TimelineMetricStore.java | 2 +- .../webapp/TimelineWebServices.java | 8 ++-- .../timeline/TestTimelineMetricStore.java | 2 +- 5 files changed, 56 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8a731458/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index fa095a0..c242a2f 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -51,6 +52,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -388,8 +390,50 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin } @Override - public Map<String, Set<String>> getInstanceHostsMetadata() throws SQLException, IOException { - return metricMetadataManager.getHostedInstanceCache(); + public Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) + throws SQLException, IOException { + + Map<String, Set<String>> hostedApps = metricMetadataManager.getHostedAppsCache(); + Map<String, Set<String>> instanceHosts = metricMetadataManager.getHostedInstanceCache(); + Map<String, Map<String, Set<String>>> instanceAppHosts = new HashMap<>(); + + if (MapUtils.isEmpty(instanceHosts)) { + Map<String, Set<String>> appHostMap = new HashMap<String, Set<String>>(); + for (String host : hostedApps.keySet()) { + for (String app : hostedApps.get(host)) { + if (!appHostMap.containsKey(app)) { + appHostMap.put(app, new HashSet<String>()); + } + appHostMap.get(app).add(host); + } + } + instanceAppHosts.put("", appHostMap); + } else { + for (String instance : instanceHosts.keySet()) { + + if (StringUtils.isNotEmpty(instanceId) && !instance.equals(instanceId)) { + continue; + } + Map<String, Set<String>> appHostMap = new HashMap<String, Set<String>>(); + instanceAppHosts.put(instance, appHostMap); + + Set<String> hostsWithInstance = instanceHosts.get(instance); + for (String host : hostsWithInstance) { + for (String app : hostedApps.get(host)) { + if (StringUtils.isNotEmpty(appId) && !app.equals(appId)) { + continue; + } + + if (!appHostMap.containsKey(app)) { + appHostMap.put(app, new HashSet<String>()); + } + appHostMap.get(app).add(host); + } + } + } + } + + return instanceAppHosts; } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/8a731458/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java index 65bbc4c..fb369e8 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java @@ -787,7 +787,9 @@ public class PhoenixHBaseAccessor { metadataManager.putIfModifiedHostedAppsMetadata( tm.getHostName(), tm.getAppId()); - metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(), tm.getHostName()); + if (!tm.getAppId().equals("FLUME_HANDLER")) { + metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(), tm.getHostName()); + } } if (!acceptMetric) { iterator.remove(); http://git-wip-us.apache.org/repos/asf/ambari/blob/8a731458/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java index 121a8ae..bde09cb 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java @@ -94,7 +94,7 @@ public interface TimelineMetricStore { * @throws SQLException * @throws IOException */ - Map<String, Set<String>> getInstanceHostsMetadata() throws SQLException, IOException; + Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) throws SQLException, IOException; /** * Return a list of known live collector nodes http://git-wip-us.apache.org/repos/asf/ambari/blob/8a731458/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java index 6278c59..9da921a 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java @@ -415,14 +415,16 @@ public class TimelineWebServices { @GET @Path("/metrics/instances") @Produces({ MediaType.APPLICATION_JSON }) - public Map<String, Set<String>> getClusterHostsMetadata( + public Map<String, Map<String, Set<String>>> getClusterHostsMetadata( @Context HttpServletRequest req, - @Context HttpServletResponse res + @Context HttpServletResponse res, + @QueryParam("appId") String appId, + @QueryParam("instanceId") String instanceId ) { init(res); try { - return timelineMetricStore.getInstanceHostsMetadata(); + return timelineMetricStore.getInstanceHostsMetadata(instanceId, appId); } catch (Exception e) { throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); } http://git-wip-us.apache.org/repos/asf/ambari/blob/8a731458/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java index b40481d..f00906e 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java @@ -97,7 +97,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore { } @Override - public Map<String, Set<String>> getInstanceHostsMetadata() throws SQLException, IOException { + public Map<String, Map<String,Set<String>>> getInstanceHostsMetadata(String instanceId, String appId) throws SQLException, IOException { return Collections.emptyMap(); }
