AMBARI-10196. Expose metadata for the metrics via API. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d83c14c4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d83c14c4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d83c14c4 Branch: refs/heads/trunk Commit: d83c14c401d90e6355283a23d17b41b5d778fd28 Parents: d95dfe3 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Tue Mar 24 19:20:55 2015 -0700 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Tue Mar 24 19:20:55 2015 -0700 ---------------------------------------------------------------------- .../server/api/services/AmbariMetaInfo.java | 26 +- .../internal/StackArtifactResourceProvider.java | 139 +- .../server/state/stack/MetricDefinition.java | 26 +- .../AMBARI_METRICS/0.1.0/metrics.json | 2946 +- .../FLUME/1.4.0.2.0/metrics.json | 810 +- .../HBASE/0.96.0.2.0/metrics.json | 26360 ++++++++-------- .../common-services/HDFS/2.1.0.2.0/metrics.json | 14720 ++++----- .../KAFKA/0.8.1.2.2/metrics.json | 466 +- .../STORM/0.9.1.2.1/metrics.json | 16 +- .../common-services/YARN/2.1.0.2.0/metrics.json | 9558 +++--- .../src/main/resources/ganglia_properties.json | 4 +- .../BIGTOP/0.8/services/FLUME/metrics.json | 1390 +- .../BIGTOP/0.8/services/HBASE/metrics.json | 27130 +++++++++-------- .../BIGTOP/0.8/services/HDFS/metrics.json | 15540 +++++----- .../BIGTOP/0.8/services/YARN/metrics.json | 10398 ++++--- .../2.0.6.GlusterFS/services/YARN/metrics.json | 4970 +-- .../2.1.GlusterFS/services/STORM/metrics.json | 2014 +- .../2.1.GlusterFS/services/YARN/metrics.json | 4970 +-- .../2.2.GlusterFS/services/KAFKA/metrics.json | 466 +- .../2.2.GlusterFS/services/YARN/metrics.json | 4970 +-- .../stacks/HDP/2.2/services/STORM/metrics.json | 2022 +- .../PHD/3.0.0.0/services/FLUME/metrics.json | 1390 +- .../PHD/3.0.0.0/services/HBASE/metrics.json | 27130 +++++++++-------- .../PHD/3.0.0.0/services/HDFS/metrics.json | 15540 +++++----- .../PHD/3.0.0.0/services/YARN/metrics.json | 10398 ++++--- .../StackArtifactResourceProviderTest.java | 123 + .../HDP/2.0.5/services/HBASE/metrics.json | 6340 ++-- .../stacks/HDP/2.0.5/services/HDFS/metrics.json | 5742 ++-- .../stacks/HDP/2.0.5/services/YARN/metrics.json | 3250 +- .../stacks/HDP/2.0.8/services/HDFS/metrics.json | 15540 +++++----- .../HDP/2.1.1/services/STORM/metrics.json | 166 +- .../stacks/OTHER/1.0/services/HDFS/metrics.json | 15540 +++++----- .../OTHER/1.0/services/STORM/metrics.json | 192 +- .../OTHER/1.0/services/HDFS/metrics.json | 15540 +++++----- 34 files changed, 124182 insertions(+), 121650 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d83c14c4/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 982f10f..3ac2d0d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -229,7 +229,7 @@ public class AmbariMetaInfo { // Need to be initialized before all actions ALL_SUPPORTED_OS = new ArrayList<String>(os_family.os_list()); readServerVersion(); - stackManager = new StackManager(stackRoot,commonServicesRoot, + stackManager = new StackManager(stackRoot, commonServicesRoot, new StackContext(metaInfoDAO, actionMetadata, os_family)); getCustomActionDefinitions(customActionRoot); } @@ -800,12 +800,10 @@ public class AmbariMetaInfo { } /** - * Gets the metrics for a Role (component). - * @return the list of defined metrics. + * Return metrics for a stack service. */ - public List<MetricDefinition> getMetrics(String stackName, String stackVersion, - String serviceName, String componentName, String metricType) - throws AmbariException { + public Map<String, Map<String, List<MetricDefinition>>> getServiceMetrics(String stackName, + String stackVersion, String serviceName) throws AmbariException { ServiceInfo svc = getService(stackName, stackVersion, serviceName); @@ -836,7 +834,21 @@ public class AmbariMetaInfo { } } - if (map.containsKey(componentName)) { + return map; + } + + /** + * Gets the metrics for a Role (component). + * @return the list of defined metrics. + */ + public List<MetricDefinition> getMetrics(String stackName, String stackVersion, + String serviceName, String componentName, String metricType) + throws AmbariException { + + Map<String, Map<String, List<MetricDefinition>>> map = + getServiceMetrics(stackName, stackVersion, serviceName); + + if (map != null && map.containsKey(componentName)) { if (map.get(componentName).containsKey(metricType)) { return map.get(componentName).get(metricType); } http://git-wip-us.apache.org/repos/asf/ambari/blob/d83c14c4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackArtifactResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackArtifactResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackArtifactResourceProvider.java index 8a6e38d..44828fa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackArtifactResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackArtifactResourceProvider.java @@ -22,6 +22,7 @@ import com.google.inject.Inject; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.StaticallyInject; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; import org.apache.ambari.server.controller.spi.NoSuchResourceException; @@ -40,11 +41,13 @@ import org.apache.ambari.server.state.kerberos.KerberosDescriptor; import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory; import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptorFactory; +import org.apache.ambari.server.state.stack.MetricDefinition; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -82,6 +85,12 @@ public class StackArtifactResourceProvider extends AbstractControllerResourcePro PropertyHelper.getPropertyId("Artifacts", "service_name"); /** + * stack service name + */ + public static final String STACK_COMPONENT_NAME_PROPERTY_ID = + PropertyHelper.getPropertyId("Artifacts", "component_name"); + + /** * artifact name */ public static final String ARTIFACT_NAME_PROPERTY_ID = @@ -118,6 +127,15 @@ public class StackArtifactResourceProvider extends AbstractControllerResourcePro */ public static final String THEME_ARTIFACT_NAME = "theme"; + /** + * name of the metrics descriptor artifact. + */ + public static final String METRICS_DESCRIPTOR_NAME = "metrics_descriptor"; + + /** + * name of the widgets descriptor artifact. + */ + public static final String WIDGETS_DESCRIPTOR_NAME = "widgets_descriptor"; /** * KerberosDescriptorFactory used to create KerberosDescriptor instances @@ -171,6 +189,7 @@ public class StackArtifactResourceProvider extends AbstractControllerResourcePro Set<Resource> resources = new HashSet<Resource>(); resources.addAll(getKerberosDescriptors(request, predicate)); + resources.addAll(getMetricsDescriptors(request, predicate)); resources.addAll(getThemes(request, predicate)); // add other artifacts types here @@ -218,7 +237,7 @@ public class StackArtifactResourceProvider extends AbstractControllerResourcePro } /** - * Get all stack and stack service descriptor resources. + * Get all stack and stack service kerberos descriptor resources. * * @param request user request * @param predicate request predicate @@ -329,6 +348,124 @@ public class StackArtifactResourceProvider extends AbstractControllerResourcePro } /** + * Get all stack and stack service metrics descriptor resources. + * + * @param request user request + * @param predicate request predicate + * + * @return set of all stack related kerberos descriptor resources; will not return null + * + * @throws SystemException if an unexpected exception occurs + * @throws UnsupportedPropertyException if an unsupported property was requested + * @throws NoSuchParentResourceException if a specified parent resource doesn't exist + * @throws NoSuchResourceException if the requested resource doesn't exist + */ + private Set<Resource> getMetricsDescriptors(Request request, Predicate predicate) + throws SystemException, UnsupportedPropertyException, + NoSuchParentResourceException, NoSuchResourceException { + + Set<Resource> resources = new HashSet<Resource>(); + + for (Map<String, Object> properties : getPropertyMaps(predicate)) { + String artifactName = (String) properties.get(ARTIFACT_NAME_PROPERTY_ID); + if (artifactName == null || artifactName.equals(METRICS_DESCRIPTOR_NAME)) { + String stackName = (String) properties.get(STACK_NAME_PROPERTY_ID); + String stackVersion = (String) properties.get(STACK_VERSION_PROPERTY_ID); + String stackService = (String) properties.get(STACK_SERVICE_NAME_PROPERTY_ID); + String componentName = (String) properties.get(STACK_COMPONENT_NAME_PROPERTY_ID); + + Map<String, Object> descriptor; + AmbariMetaInfo metaInfo = getManagementController().getAmbariMetaInfo(); + + try { + List<MetricDefinition> componentMetrics; + Map<String, Map<String, List<MetricDefinition>>> serviceMetrics; + if (stackService != null) { + if (componentName == null) { + // Service + serviceMetrics = metaInfo.getServiceMetrics(stackName, stackVersion, stackService); + descriptor = Collections.singletonMap(stackService, (Object) serviceMetrics); + } else { + // Component + componentMetrics = metaInfo.getMetrics(stackName, stackVersion, stackService, componentName, Resource.Type.Component.name()); + descriptor = Collections.singletonMap(componentName, (Object) componentMetrics); + } + } else { + // Cluster + Map<String, Map<String, PropertyInfo>> clusterMetrics = + PropertyHelper.getMetricPropertyIds(Resource.Type.Cluster); + // Host + Map<String, Map<String, PropertyInfo>> hostMetrics = + PropertyHelper.getMetricPropertyIds(Resource.Type.Host); + + descriptor = new HashMap<String, Object>(); + descriptor.put(Resource.Type.Cluster.name(), clusterMetrics); + descriptor.put(Resource.Type.Host.name(), hostMetrics); + } + + + } catch (IOException e) { + LOG.error("Unable to process Kerberos Descriptor. Properties: " + properties, e); + throw new SystemException("An internal exception occurred while attempting to build a Kerberos Descriptor " + + "artifact. See ambari server logs for more information", e); + } + + Resource resource = new ResourceImpl(Resource.Type.StackArtifact); + Set<String> requestedIds = getRequestPropertyIds(request, predicate); + setResourceProperty(resource, ARTIFACT_NAME_PROPERTY_ID, METRICS_DESCRIPTOR_NAME, requestedIds); + setResourceProperty(resource, ARTIFACT_DATA_PROPERTY_ID, descriptor, requestedIds); + setResourceProperty(resource, STACK_NAME_PROPERTY_ID, stackName, requestedIds); + setResourceProperty(resource, STACK_VERSION_PROPERTY_ID, stackVersion, requestedIds); + if (stackService != null) { + setResourceProperty(resource, STACK_SERVICE_NAME_PROPERTY_ID, stackService, requestedIds); + } + resources.add(resource); + } + } + return resources; + } + + private Set<Resource> getWidgetsDescriptors(Request request, Predicate predicate) + throws SystemException, UnsupportedPropertyException, + NoSuchParentResourceException, NoSuchResourceException { + + Set<Resource> resources = new HashSet<Resource>(); + + for (Map<String, Object> properties : getPropertyMaps(predicate)) { + String artifactName = (String) properties.get(ARTIFACT_NAME_PROPERTY_ID); + if (artifactName == null || artifactName.equals(KERBEROS_DESCRIPTOR_NAME)) { + String stackName = (String) properties.get(STACK_NAME_PROPERTY_ID); + String stackVersion = (String) properties.get(STACK_VERSION_PROPERTY_ID); + String stackService = (String) properties.get(STACK_SERVICE_NAME_PROPERTY_ID); + + Map<String, Object> descriptor; + try { + descriptor = getKerberosDescriptor(stackName, stackVersion, stackService); + } catch (IOException e) { + LOG.error("Unable to process Kerberos Descriptor. Properties: " + properties, e); + throw new SystemException("An internal exception occurred while attempting to build a Kerberos Descriptor " + + "artifact. See ambari server logs for more information", e); + } + + if (descriptor != null) { + Resource resource = new ResourceImpl(Resource.Type.StackArtifact); + Set<String> requestedIds = getRequestPropertyIds(request, predicate); + setResourceProperty(resource, ARTIFACT_NAME_PROPERTY_ID, KERBEROS_DESCRIPTOR_NAME, requestedIds); + setResourceProperty(resource, ARTIFACT_DATA_PROPERTY_ID, descriptor, requestedIds); + setResourceProperty(resource, STACK_NAME_PROPERTY_ID, stackName, requestedIds); + setResourceProperty(resource, STACK_VERSION_PROPERTY_ID, stackVersion, requestedIds); + if (stackService != null) { + setResourceProperty(resource, STACK_SERVICE_NAME_PROPERTY_ID, stackService, requestedIds); + } + resources.add(resource); + } + } + } + return resources; + } + + + /** * Get a kerberos descriptor. * * @param stackName stack name http://git-wip-us.apache.org/repos/asf/ambari/blob/d83c14c4/ambari-server/src/main/java/org/apache/ambari/server/state/stack/MetricDefinition.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/MetricDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/MetricDefinition.java index d0f517b..a95356c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/MetricDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/MetricDefinition.java @@ -18,16 +18,21 @@ package org.apache.ambari.server.state.stack; +import org.codehaus.jackson.annotate.JsonIgnore; +import org.codehaus.jackson.annotate.JsonProperty; + import java.util.HashMap; import java.util.Map; +import static java.util.Map.Entry; + /** * Used to represent metrics for a stack component. */ public class MetricDefinition { private String type = null; private Map<String, String> properties = null; - private Map<String, Metric> metrics = null; + private Map<String, Map<String, Metric>> metrics = null; public String getType() { return type; @@ -36,16 +41,29 @@ public class MetricDefinition { public Map<String, String> getProperties() { return properties; } - - public Map<String, Metric> getMetrics() { + + @JsonProperty("metrics") + public Map<String, Map<String, Metric>> getMetricsByCategory() { return metrics; } + + /** + * Return flat metric map without category + */ + @JsonIgnore + public Map<String, Metric> getMetrics() { + Map<String, Metric> metricMap = new HashMap<String, Metric>(); + for (Entry<String, Map<String, Metric>> metricMapEntry : metrics.entrySet()) { + metricMap.putAll(metricMapEntry.getValue()); + } + return metricMap; + } public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{type=").append(type); sb.append(";properties=").append(properties); - sb.append(";metric_count=").append(metrics.size()); + sb.append(";metric_count=").append(getMetrics().size()); sb.append('}'); return sb.toString();
