Repository: ambari Updated Branches: refs/heads/branch-2.5 47871b151 -> 176de64d3
AMBARI-20820 Stack side changes to use instanceId for cluster based segregation of data (AMS) (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/176de64d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/176de64d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/176de64d Branch: refs/heads/branch-2.5 Commit: 176de64d3b827dfb807b7c4decb312bc4de2721f Parents: 47871b1 Author: Dmytro Sen <d...@apache.org> Authored: Wed Apr 26 21:38:29 2017 +0300 Committer: Dmytro Sen <d...@apache.org> Committed: Wed Apr 26 21:38:29 2017 +0300 ---------------------------------------------------------------------- .../timeline/AbstractTimelineMetricsSink.java | 2 ++ .../sink/flume/FlumeTimelineMetricsSink.java | 10 +++++- .../timeline/HadoopTimelineMetricsSink.java | 9 +++-- .../timeline/HadoopTimelineMetricsSinkTest.java | 14 +++++++- .../main/python/core/application_metric_map.py | 7 ++-- .../src/main/python/core/config_reader.py | 6 ++++ .../src/main/python/core/emitter.py | 4 ++- .../kafka/KafkaTimelineMetricsReporter.java | 10 ++++++ .../kafka/KafkaTimelineMetricsReporterTest.java | 4 +++ .../storm/StormTimelineMetricsReporter.java | 7 ++++ .../sink/storm/StormTimelineMetricsSink.java | 9 ++++- .../storm/StormTimelineMetricsReporter.java | 8 ++++- .../sink/storm/StormTimelineMetricsSink.java | 8 ++++- .../system/impl/AmbariMetricSinkImpl.java | 38 ++++++++++++-------- .../1.6.1.2.2.0/package/scripts/params.py | 15 ++++++-- .../hadoop-metrics2-accumulo.properties.j2 | 2 ++ .../0.1.0/package/scripts/params.py | 13 ++++--- .../hadoop-metrics2-hbase.properties.j2 | 2 ++ .../package/templates/metric_monitor.ini.j2 | 2 ++ .../FLUME/1.4.0.2.0/package/scripts/params.py | 21 ++++++----- .../templates/flume-metrics2.properties.j2 | 3 ++ .../0.96.0.2.0/package/scripts/params_linux.py | 14 ++++++-- ...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 2 ++ ...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 3 +- .../0.12.0.2.0/package/scripts/params_linux.py | 17 ++++++--- .../hadoop-metrics2-hivemetastore.properties.j2 | 2 ++ .../hadoop-metrics2-hiveserver2.properties.j2 | 2 ++ .../templates/hadoop-metrics2-llapdaemon.j2 | 4 ++- .../hadoop-metrics2-llaptaskscheduler.j2 | 4 ++- .../KAFKA/0.8.1/configuration/kafka-broker.xml | 10 ++++++ .../KAFKA/0.8.1/package/scripts/params.py | 15 ++++++-- .../STORM/0.9.1/package/scripts/params_linux.py | 12 +++++-- .../0.9.1/package/templates/config.yaml.j2 | 3 ++ .../templates/storm-metrics2.properties.j2 | 2 ++ .../2.0.6/hooks/before-START/scripts/params.py | 13 +++++-- .../templates/hadoop-metrics2.properties.j2 | 2 ++ .../hadoop-metrics2.properties.xml | 2 ++ .../python/stacks/2.0.6/configs/default.json | 4 +-- 38 files changed, 243 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java index 46f32f9..2c6fae2 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java @@ -79,6 +79,8 @@ public abstract class AbstractTimelineMetricsSink { public static final String SSL_KEYSTORE_TYPE_PROPERTY = "truststore.type"; public static final String SSL_KEYSTORE_PASSWORD_PROPERTY = "truststore.password"; public static final String COLLECTOR_LIVE_NODES_PATH = "/ws/v1/timeline/metrics/livenodes"; + public static final String INSTANCE_ID_PROPERTY = "instanceId"; + public static final String SET_INSTANCE_ID_PROPERTY = "set.instanceId"; protected static final AtomicInteger failedCollectorConnectionsCounter = new AtomicInteger(0); public static int NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS = 100; http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java index 86e092a..a8318c2 100644 --- a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java @@ -61,6 +61,8 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem private final static String COUNTER_METRICS_PROPERTY = "counters"; private final Set<String> counterMetrics = new HashSet<String>(); private int timeoutSeconds = 10; + private boolean setInstanceId; + private String instanceId; @Override public void start() { @@ -104,6 +106,8 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem zookeeperQuorum = configuration.getProperty("zookeeper.quorum"); protocol = configuration.getProperty(COLLECTOR_PROTOCOL, "http"); port = configuration.getProperty(COLLECTOR_PORT, "6188"); + setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY, "false")); + instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY, ""); // Initialize the collector write strategy super.init(); @@ -225,7 +229,11 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem TimelineMetric timelineMetric = new TimelineMetric(); timelineMetric.setMetricName(attributeName); timelineMetric.setHostName(hostname); - timelineMetric.setInstanceId(component); + if (setInstanceId) { + timelineMetric.setInstanceId(instanceId + component); + } else { + timelineMetric.setInstanceId(component); + } timelineMetric.setAppId("FLUME_HANDLER"); timelineMetric.setStartTime(currentTimeMillis); timelineMetric.getMetricValues().put(currentTimeMillis, Double.parseDouble(attributeValue)); http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java index 85cc794..bcdc65a 100644 --- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java @@ -52,6 +52,7 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple private TimelineMetricsCache metricsCache; private String hostName = "UNKNOWN.example.com"; private String instanceId = null; + private boolean setInstanceId; private String serviceName = ""; private Collection<String> collectorHosts; private String collectorUri; @@ -95,8 +96,8 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple } serviceName = getServiceName(conf); - String inst = conf.getString("instanceId", ""); - instanceId = StringUtils.isEmpty(inst) ? null : inst; + instanceId = conf.getString(INSTANCE_ID_PROPERTY); + setInstanceId = conf.getBoolean(SET_INSTANCE_ID_PROPERTY, false); LOG.info("Identified hostname = " + hostName + ", serviceName = " + serviceName); // Initialize the collector write strategy @@ -322,7 +323,9 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple timelineMetric.setMetricName(name); timelineMetric.setHostName(hostName); timelineMetric.setAppId(serviceName); - timelineMetric.setInstanceId(instanceId); + if (setInstanceId) { + timelineMetric.setInstanceId(instanceId); + } timelineMetric.setStartTime(startTime); timelineMetric.setType(metric.type() != null ? metric.type().name() : null); timelineMetric.getMetricValues().put(startTime, value.doubleValue()); http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java index 0cafd9a..7fe2861 100644 --- a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java @@ -51,12 +51,15 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.TreeMap; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_PORT; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_HOSTS_PROPERTY; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_PROTOCOL; +import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.INSTANCE_ID_PROPERTY; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.MAX_METRIC_ROW_CACHE_SIZE; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.METRICS_SEND_INTERVAL; +import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.SET_INSTANCE_ID_PROPERTY; import static org.easymock.EasyMock.anyInt; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.anyString; @@ -82,13 +85,17 @@ public class HadoopTimelineMetricsSinkTest { } @Test - @PrepareForTest({URL.class, OutputStream.class, AbstractTimelineMetricsSink.class, HttpURLConnection.class}) + @PrepareForTest({URL.class, OutputStream.class, AbstractTimelineMetricsSink.class, HttpURLConnection.class, TimelineMetric.class, HadoopTimelineMetricsSink.class}) public void testPutMetrics() throws Exception { HadoopTimelineMetricsSink sink = new HadoopTimelineMetricsSink(); HttpURLConnection connection = PowerMock.createNiceMock(HttpURLConnection.class); URL url = PowerMock.createNiceMock(URL.class); InputStream is = IOUtils.toInputStream(gson.toJson(Collections.singletonList("localhost"))); + TimelineMetric timelineMetric = PowerMock.createNiceMock(TimelineMetric.class); + expectNew(TimelineMetric.class).andReturn(timelineMetric).times(2); + expect(timelineMetric.getMetricValues()).andReturn(new TreeMap<Long, Double>()).anyTimes(); + expect(timelineMetric.getMetricName()).andReturn("metricName").anyTimes(); expectNew(URL.class, anyString()).andReturn(url).anyTimes(); expect(url.openConnection()).andReturn(connection).anyTimes(); expect(connection.getInputStream()).andReturn(is).anyTimes(); @@ -107,6 +114,8 @@ public class HadoopTimelineMetricsSinkTest { expect(conf.getInt(eq(MAX_METRIC_ROW_CACHE_SIZE), anyInt())).andReturn(10).anyTimes(); expect(conf.getInt(eq(METRICS_SEND_INTERVAL), anyInt())).andReturn(1000).anyTimes(); + expect(conf.getBoolean(eq(SET_INSTANCE_ID_PROPERTY), eq(false))).andReturn(true).anyTimes(); + expect(conf.getString(eq(INSTANCE_ID_PROPERTY))).andReturn("instanceId").anyTimes(); conf.setListDelimiter(eq(',')); expectLastCall().anyTimes(); @@ -146,6 +155,9 @@ public class HadoopTimelineMetricsSinkTest { expect(record.metrics()).andReturn(Arrays.asList(metric)).anyTimes(); + timelineMetric.setInstanceId(eq("instanceId")); + EasyMock.expectLastCall(); + replay(conf, record, metric); replayAll(); http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py index 0052808..34a6787 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py @@ -66,7 +66,7 @@ class ApplicationMetricMap: del self.app_metric_map[ app_id ] pass - def flatten(self, application_id = None, clear_once_flattened = False): + def flatten(self, application_id = None, clear_once_flattened = False, set_instanceid = False, instanceid = None): """ Return flatten dict to caller in json format. Json format: @@ -89,11 +89,14 @@ class ApplicationMetricMap: for appId, metrics in local_metric_map.iteritems(): for metricId, metricData in dict(metrics).iteritems(): # Create a timeline metric object + result_instanceid = "" + if set_instanceid: + result_instanceid = instanceid timeline_metric = { "hostname" : self.hostname, "metricname" : metricId, "appid" : "HOST", - "instanceid" : "", + "instanceid" : result_instanceid, "starttime" : self.get_start_time(appId, metricId), "metrics" : metricData } http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py index 7010187..4fe0c4c 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py @@ -240,9 +240,15 @@ class Configuration: def is_server_https_enabled(self): return "true" == str(self.get("collector", "https_enabled")).lower() + def is_set_instanceid(self): + return "true" == str(self.get("default", "set.instanceId", 'false')).lower() + def get_server_host(self): return self.get("collector", "host") + def get_instanceid(self): + return self.get("default", "instanceid") + def get_server_port(self): try: return int(self.get("collector", "port")) http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py index 6dc0207..232668e 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py @@ -46,6 +46,8 @@ class Emitter(threading.Thread): self.collector_port = config.get_server_port() self.all_metrics_collector_hosts = config.get_metrics_collector_hosts() self.is_server_https_enabled = config.is_server_https_enabled() + self.set_instanceid = config.is_set_instanceid() + self.instanceid = config.get_instanceid() if self.is_server_https_enabled: self.ca_certs = config.get_ca_certs() @@ -74,7 +76,7 @@ class Emitter(threading.Thread): # This call will acquire lock on the map and clear contents before returning # After configured number of retries the data will not be sent to the # collector - json_data = self.application_metric_map.flatten(None, True) + json_data = self.application_metric_map.flatten(None, True, set_instanceid=self.set_instanceid, instanceid=self.instanceid) if json_data is None: logger.info("Nothing to emit, resume waiting.") return http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java index 9d492cb..211e9cd 100644 --- a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java +++ b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java @@ -70,6 +70,8 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink private static final String TIMELINE_METRICS_SSL_KEYSTORE_PATH_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + SSL_KEYSTORE_PATH_PROPERTY; private static final String TIMELINE_METRICS_SSL_KEYSTORE_TYPE_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + SSL_KEYSTORE_TYPE_PROPERTY; private static final String TIMELINE_METRICS_SSL_KEYSTORE_PASSWORD_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + SSL_KEYSTORE_PASSWORD_PROPERTY; + private static final String TIMELINE_METRICS_KAFKA_INSTANCE_ID_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + INSTANCE_ID_PROPERTY; + private static final String TIMELINE_METRICS_KAFKA_SET_INSTANCE_ID_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + SET_INSTANCE_ID_PROPERTY; private static final String TIMELINE_DEFAULT_HOST = "localhost"; private static final String TIMELINE_DEFAULT_PORT = "6188"; private static final String TIMELINE_DEFAULT_PROTOCOL = "http"; @@ -87,6 +89,8 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink private TimelineMetricsCache metricsCache; private int timeoutSeconds = 10; private String zookeeperQuorum = null; + private boolean setInstanceId; + private String instanceId; private String[] excludedMetricsPrefixes; private String[] includedMetricsPrefixes; @@ -162,6 +166,9 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink collectorHosts = parseHostsStringIntoCollection(props.getString(TIMELINE_HOSTS_PROPERTY, TIMELINE_DEFAULT_HOST)); metricCollectorProtocol = props.getString(TIMELINE_PROTOCOL_PROPERTY, TIMELINE_DEFAULT_PROTOCOL); + instanceId = props.getString(TIMELINE_METRICS_KAFKA_INSTANCE_ID_PROPERTY); + setInstanceId = props.getBoolean(TIMELINE_METRICS_KAFKA_SET_INSTANCE_ID_PROPERTY); + setMetricsCache(new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval)); if (metricCollectorProtocol.contains("https")) { @@ -315,6 +322,9 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink TimelineMetric timelineMetric = new TimelineMetric(); timelineMetric.setMetricName(attributeName); timelineMetric.setHostName(hostname); + if (setInstanceId) { + timelineMetric.setInstanceId(instanceId); + } timelineMetric.setAppId(component); timelineMetric.setStartTime(currentTimeMillis); timelineMetric.setType(ClassUtils.getShortCanonicalName(attributeValue, "Number")); http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java b/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java index e1ac48c..b05190c 100644 --- a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java +++ b/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java @@ -84,6 +84,8 @@ public class KafkaTimelineMetricsReporterTest { properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true"); properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c"); properties.setProperty("external.kafka.metrics.include.prefix", "a.b.c.d"); + properties.setProperty("kafka.timeline.metrics.instanceId", "cluster"); + properties.setProperty("kafka.timeline.metrics.set.instanceId", "false"); props = new VerifiableProperties(properties); } @@ -120,6 +122,8 @@ public class KafkaTimelineMetricsReporterTest { properties.setProperty("kafka.timeline.metrics.truststore.path", ""); properties.setProperty("kafka.timeline.metrics.truststore.type", ""); properties.setProperty("kafka.timeline.metrics.truststore.password", ""); + properties.setProperty("kafka.timeline.metrics.instanceId", "cluster"); + properties.setProperty("kafka.timeline.metrics.set.instanceId", "false"); kafkaTimelineMetricsReporter.init(new VerifiableProperties(properties)); kafkaTimelineMetricsReporter.stopReporter(); verifyAll(); http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java index 9e216fc..e550024 100644 --- a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java +++ b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java @@ -53,6 +53,8 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink private Collection<String> collectorHosts; private String zkQuorum; private String protocol; + private boolean setInstanceId; + private String instanceId; public StormTimelineMetricsReporter() { @@ -133,6 +135,8 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink Integer.parseInt(cf.get(METRICS_POST_TIMEOUT_SECONDS).toString()) : DEFAULT_POST_TIMEOUT_SECONDS; applicationId = cf.get(APP_ID).toString(); + setInstanceId = Boolean.getBoolean(cf.get(SET_INSTANCE_ID_PROPERTY).toString()); + instanceId = cf.get(INSTANCE_ID_PROPERTY).toString(); } catch (Exception e) { LOG.warn("Could not initialize metrics collector, please specify " + @@ -195,6 +199,9 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink TimelineMetric timelineMetric = new TimelineMetric(); timelineMetric.setMetricName(attributeName); timelineMetric.setHostName(hostname); + if (setInstanceId) { + timelineMetric.setInstanceId(instanceId); + } timelineMetric.setAppId(component); timelineMetric.setStartTime(currentTimeMillis); timelineMetric.getMetricValues().put(currentTimeMillis, Double.parseDouble(attributeValue)); http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java index 8029e05..20f60e1 100644 --- a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java @@ -59,6 +59,8 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem private String port; private String topologyName; private String applicationId; + private boolean setInstanceId; + private String instanceId; @Override protected String getCollectorUri(String host) { @@ -122,6 +124,8 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem protocol = configuration.getProperty(COLLECTOR_PROTOCOL, "http"); port = configuration.getProperty(COLLECTOR_PORT, "6188"); + instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY); + setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY, "false")); // Initialize the collector write strategy super.init(); @@ -243,6 +247,9 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem TimelineMetric timelineMetric = new TimelineMetric(); timelineMetric.setMetricName(attributeName); timelineMetric.setHostName(hostName); + if (setInstanceId) { + timelineMetric.setInstanceId(instanceId); + } timelineMetric.setAppId(applicationId); timelineMetric.setStartTime(currentTimeMillis); timelineMetric.setType(ClassUtils.getShortCanonicalName( @@ -301,4 +308,4 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem this.metricsCache = metricsCache; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java index b716c7e..cebaf93 100644 --- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java +++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java @@ -48,6 +48,8 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink private Collection<String> collectorHosts; private String zkQuorum; private String protocol; + private boolean setInstanceId; + private String instanceId; public StormTimelineMetricsReporter() { @@ -115,7 +117,8 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink Integer.parseInt(conf.getProperty(METRICS_POST_TIMEOUT_SECONDS).toString()) : DEFAULT_POST_TIMEOUT_SECONDS; applicationId = conf.getProperty(CLUSTER_REPORTER_APP_ID, DEFAULT_CLUSTER_REPORTER_APP_ID); - + setInstanceId = Boolean.valueOf(conf.getProperty(SET_INSTANCE_ID_PROPERTY)); + instanceId = conf.getProperty(INSTANCE_ID_PROPERTY); if (protocol.contains("https")) { String trustStorePath = conf.getProperty(SSL_KEYSTORE_PATH_PROPERTY).toString().trim(); String trustStoreType = conf.getProperty(SSL_KEYSTORE_TYPE_PROPERTY).toString().trim(); @@ -229,6 +232,9 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink TimelineMetric timelineMetric = new TimelineMetric(); timelineMetric.setMetricName(attributeName); timelineMetric.setHostName(hostname); + if (setInstanceId) { + timelineMetric.setInstanceId(instanceId); + } timelineMetric.setAppId(component); timelineMetric.setStartTime(currentTimeMillis); timelineMetric.setType(ClassUtils.getShortCanonicalName(attributeValue, "Number")); http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java index f58f549..425201c 100644 --- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java @@ -68,6 +68,8 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem private String port; private String topologyName; private String applicationId; + private String instanceId; + private boolean setInstanceId; @Override protected String getCollectorUri(String host) { @@ -133,7 +135,8 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem protocol = configuration.getProperty(COLLECTOR_PROTOCOL, "http"); port = configuration.getProperty(COLLECTOR_PORT, "6188"); - + instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY); + setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY, "false")); // Initialize the collector write strategy super.init(); @@ -332,6 +335,9 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem TimelineMetric timelineMetric = new TimelineMetric(); timelineMetric.setMetricName(attributeName); timelineMetric.setHostName(hostName); + if (setInstanceId) { + timelineMetric.setInstanceId(instanceId); + } timelineMetric.setAppId(applicationId); timelineMetric.setStartTime(currentTimeMillis); timelineMetric.setType(ClassUtils.getShortCanonicalName( http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java index 336a2e2..767398b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.metrics.system.impl; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashMap; @@ -71,6 +72,8 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements private AmbariManagementController ambariManagementController; private TimelineMetricsCache timelineMetricsCache; private boolean isInitialized = false; + private boolean setInstanceId = false; + private String instanceId; public AmbariMetricSinkImpl(AmbariManagementController amc) { this.ambariManagementController = amc; @@ -98,28 +101,30 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements for (Map.Entry<String, Cluster> kv : clusters.getClusters().entrySet()) { String clusterName = kv.getKey(); + instanceId = clusterName; Cluster c = kv.getValue(); Resource.Type type = Resource.Type.ServiceConfigVersion; //If Metrics Collector VIP settings are configured, use that. - boolean vipHostConfigPresent = false; - boolean vipPortConfigPresent = false; + boolean externalHostConfigPresent = false; + boolean externalPortConfigPresent = false; Config clusterEnv = c.getDesiredConfigByType(ConfigHelper.CLUSTER_ENV); if (clusterEnv != null) { Map<String, String> configs = clusterEnv.getProperties(); - String metricsCollectorVipHost = configs.get("metrics_collector_vip_host"); - if (StringUtils.isNotEmpty(metricsCollectorVipHost)) { - LOG.info("Setting Metrics Collector Vip Host : " + metricsCollectorVipHost); - collectorHosts.add(metricsCollectorVipHost); - vipHostConfigPresent = true; + String metricsCollectorExternalHosts = configs.get("metrics_collector_external_hosts"); + if (StringUtils.isNotEmpty(metricsCollectorExternalHosts)) { + LOG.info("Setting Metrics Collector External Host : " + metricsCollectorExternalHosts); + collectorHosts.addAll(Arrays.asList(metricsCollectorExternalHosts.split(","))); + externalHostConfigPresent = true; + setInstanceId = true; } - String metricsCollectorVipPort = configs.get("metrics_collector_vip_port"); - if (StringUtils.isNotEmpty(metricsCollectorVipPort)) { - LOG.info("Setting Metrics Collector Vip Port : " + metricsCollectorVipPort); - port = metricsCollectorVipPort; - vipPortConfigPresent = true; + String metricsCollectorExternalPort = configs.get("metrics_collector_external_port"); + if (StringUtils.isNotEmpty(metricsCollectorExternalPort)) { + LOG.info("Setting Metrics Collector External Port : " + metricsCollectorExternalPort); + port = metricsCollectorExternalPort; + externalPortConfigPresent = true; } } @@ -140,7 +145,7 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements ambariManagementController); try { - if ( !vipHostConfigPresent ) { + if ( !externalHostConfigPresent ) { //get collector host(s) Service service = c.getService(ambariMetricsServiceName); if (service != null) { @@ -166,7 +171,7 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements if (config != null && config.get("type").equals("ams-site")) { TreeMap<Object, Object> properties = (TreeMap<Object, Object>) config.get("properties"); String timelineWebappAddress = (String) properties.get("timeline.metrics.service.webapp.address"); - if (!vipPortConfigPresent && StringUtils.isNotEmpty(timelineWebappAddress) && timelineWebappAddress.contains(":")) { + if (!externalPortConfigPresent && StringUtils.isNotEmpty(timelineWebappAddress) && timelineWebappAddress.contains(":")) { port = timelineWebappAddress.split(":")[1]; } String httpPolicy = (String) properties.get("timeline.metrics.service.http.policy"); @@ -318,10 +323,13 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements TimelineMetric timelineMetric = new TimelineMetric(); timelineMetric.setMetricName(attributeName); timelineMetric.setHostName(hostName); + if (setInstanceId) { + timelineMetric.setInstanceId(instanceId); + } timelineMetric.setAppId(component); timelineMetric.setStartTime(currentTimeMillis); timelineMetric.getMetricValues().put(currentTimeMillis, attributeValue.doubleValue()); return timelineMetric; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py index 53cf002..150b0a8 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py @@ -122,12 +122,21 @@ info_num_logs = config['configurations']['accumulo-log4j']['info_num_logs'] # metrics2 properties ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0] -ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) + +set_instanceId = "false" +cluster_name = config["clusterName"] + +if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_external_hosts' in config['configurations']['cluster-env']: + ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts'] + set_instanceId = "true" +else: + ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) has_metric_collector = not len(ams_collector_hosts) == 0 if has_metric_collector: if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_port' in config['configurations']['cluster-env']: - metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] + 'metrics_collector_external_port' in config['configurations']['cluster-env']: + metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port'] else: metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188") if metric_collector_web_address.find(':') != -1: http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 index f7926d0..6873c85 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 @@ -40,6 +40,8 @@ accumulo.sink.timeline.period={{metrics_collection_period}} accumulo.sink.timeline.sendInterval={{metrics_report_interval}}000 accumulo.sink.timeline.collector.hosts={{ams_collector_hosts}} accumulo.sink.timeline.port={{metric_collector_port}} +accumulo.sink.timeline.instanceId={{cluster_name}} +accumulo.sink.timeline.set.instanceId={{set_instanceId}} # HTTPS properties accumulo.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index c5dce8a..071882b 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -60,17 +60,22 @@ embedded_mode_multiple_instances = False if not is_ams_distributed and len(ams_collector_list) > 1: embedded_mode_multiple_instances = True +set_instanceId = "false" +cluster_name = config["clusterName"] if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_host' in config['configurations']['cluster-env']: - ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_vip_host'] + 'metrics_collector_external_hosts' in config['configurations']['cluster-env']: + ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts'] + set_instanceId = "true" +else: + ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts) random_metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts) if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_port' in config['configurations']['cluster-env']: - metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] + 'metrics_collector_external_port' in config['configurations']['cluster-env']: + metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port'] else: metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188") if metric_collector_web_address.find(':') != -1: http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 index 8c6f86f..9729bbe 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 @@ -65,6 +65,8 @@ hbase.sink.timeline.collector.hosts={{ams_collector_hosts}} hbase.sink.timeline.port={{metric_collector_port}} hbase.sink.timeline.protocol={{metric_collector_protocol}} hbase.sink.timeline.serviceName-prefix=ams +hbase.sink.timeline.instanceId={{cluster_name}} +hbase.sink.timeline.set.instanceId={{set_instanceId}} # HTTPS properties hbase.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 index 3823912..0e8544e 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 @@ -23,6 +23,8 @@ metrics_servers = {{ams_collector_hosts}} enable_time_threshold = false enable_value_threshold = false skip_disk_patterns = {{skip_disk_metrics_patterns}} +instanceId={{cluster_name}} +set.instanceId={{set_instanceId}} [emitter] send_interval = {{metrics_report_interval}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py index a44b461..4a3be21 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py @@ -85,18 +85,23 @@ hostname = None if config.has_key('hostname'): hostname = config['hostname'] -ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) +set_instanceId = "false" +cluster_name = config["clusterName"] + +if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_external_hosts' in config['configurations']['cluster-env']: + ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts'] + set_instanceId = "true" +else: + ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) + has_metric_collector = not len(ams_collector_hosts) == 0 metric_collector_port = None if has_metric_collector: + metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts) if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_host' in config['configurations']['cluster-env']: - metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host'] - else: - metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts) - if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_port' in config['configurations']['cluster-env']: - metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] + 'metrics_collector_external_port' in config['configurations']['cluster-env']: + metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port'] else: metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188") if metric_collector_web_address.find(':') != -1: http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 index cf2bd6c..b876a3d 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 @@ -24,6 +24,9 @@ collectionFrequency={{metrics_collection_period}}000 maxRowCacheSize=10000 sendInterval={{metrics_report_interval}}000 +instanceId={{cluster_name}} +set.instanceId={{set_instanceId}} + # HTTPS properties truststore.path = {{metric_truststore_path}} truststore.type = {{metric_truststore_type}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index 081fda1..43c6036 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -152,12 +152,20 @@ has_ganglia_server = not len(ganglia_server_hosts) == 0 if has_ganglia_server: ganglia_server_host = ganglia_server_hosts[0] -ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) +set_instanceId = "false" +cluster_name = config["clusterName"] + +if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_external_hosts' in config['configurations']['cluster-env']: + ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts'] + set_instanceId = "true" +else: + ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) has_metric_collector = not len(ams_collector_hosts) == 0 if has_metric_collector: if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_port' in config['configurations']['cluster-env']: - metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] + 'metrics_collector_external_port' in config['configurations']['cluster-env']: + metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port'] else: metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188") if metric_collector_web_address.find(':') != -1: http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 index e808f5d..24535c5 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 @@ -76,6 +76,8 @@ hbase.sink.timeline.sendInterval={{metrics_report_interval}}000 hbase.sink.timeline.collector.hosts={{ams_collector_hosts}} hbase.sink.timeline.protocol={{metric_collector_protocol}} hbase.sink.timeline.port={{metric_collector_port}} +hbase.sink.timeline.instanceId={{cluster_name}} +hbase.sink.timeline.set.instanceId={{set_instanceId}} # HTTPS properties hbase.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 index 7e1abbc..9076269 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 @@ -74,7 +74,8 @@ hbase.sink.timeline.sendInterval={{metrics_report_interval}}000 hbase.sink.timeline.collector.hosts={{ams_collector_hosts}} hbase.sink.timeline.protocol={{metric_collector_protocol}} hbase.sink.timeline.port={{metric_collector_port}} - +hbase.sink.timeline.instanceId={{cluster_name}} +hbase.sink.timeline.set.instanceId={{set_instanceId}} # HTTPS properties hbase.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 64cdfed..6c14c58 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -535,12 +535,21 @@ hive_site_config = dict(config['configurations']['hive-site']) ######################################################## ############# AMS related params ##################### ######################################################## -ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) + +set_instanceId = "false" + +if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_external_hosts' in config['configurations']['cluster-env']: + ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts'] + set_instanceId = "true" +else: + ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) + has_metric_collector = not len(ams_collector_hosts) == 0 if has_metric_collector: if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_port' in config['configurations']['cluster-env']: - metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] + 'metrics_collector_external_port' in config['configurations']['cluster-env']: + metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port'] else: metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188") if metric_collector_web_address.find(':') != -1: @@ -829,4 +838,4 @@ if enable_ranger_hive: ranger_hive_metastore_lookup = default('/configurations/ranger-hive-plugin-properties/ranger.service.config.param.enable.hive.metastore.lookup', False) if security_enabled: - hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower()) \ No newline at end of file + hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower()) http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 index 82f71c5..9328f9f 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 @@ -42,6 +42,8 @@ *.sink.timeline.period={{metrics_collection_period}} *.sink.timeline.sendInterval={{metrics_report_interval}}000 *.sink.timeline.slave.host.name = {{hostname}} + *.sink.timeline.instanceId={{cluster_name}} + *.sink.timeline.set.instanceId={{set_instanceId}} # HTTPS properties *.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 index f1bfa03..9a7f9dc 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 @@ -42,6 +42,8 @@ *.sink.timeline.period={{metrics_collection_period}} *.sink.timeline.sendInterval={{metrics_report_interval}}000 *.sink.timeline.slave.host.name = {{hostname}} + *.sink.timeline.instanceId={{cluster_name}} + *.sink.timeline.set.instanceId={{set_instanceId}} # HTTPS properties *.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 index d63dfb7..e9fe024 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 @@ -41,6 +41,8 @@ *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink *.sink.timeline.period={{metrics_collection_period}} *.sink.timeline.sendInterval={{metrics_report_interval}}000 + *.sink.timeline.instanceId={{cluster_name}} + *.sink.timeline.set.instanceId={{set_instanceId}} # HTTPS properties *.sink.timeline.truststore.path = {{metric_truststore_path}} @@ -51,4 +53,4 @@ llapdaemon.sink.timeline.port={{metric_collector_port}} llapdaemon.sink.timeline.protocol={{metric_collector_protocol}} -{% endif %} \ No newline at end of file +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 index 4c9c981..bd7eb0c 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 @@ -41,6 +41,8 @@ *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink *.sink.timeline.period={{metrics_collection_period}} *.sink.timeline.sendInterval={{metrics_report_interval}}000 + *.sink.timeline.instanceId={{cluster_name}} + *.sink.timeline.set.instanceId={{set_instanceId}} # HTTPS properties *.sink.timeline.truststore.path = {{metric_truststore_path}} @@ -51,4 +53,4 @@ llaptaskscheduler.sink.timeline.port={{metric_collector_port}} llaptaskscheduler.sink.timeline.protocol={{metric_collector_protocol}} -{% endif %} \ No newline at end of file +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml index 96b7750..1ada331 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml @@ -399,6 +399,16 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>kafka.timeline.metrics.instanceId</name> + <value>{{cluster_name}}</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>kafka.timeline.metrics.set.instanceId</name> + <value>{{set_instanceId}}</value> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>kafka.timeline.metrics.maxRowCacheSize</name> <value>10000</value> <description>Timeline metrics reporter send interval</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index 23c65b7..c36a10f 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -130,13 +130,22 @@ metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.trust metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "") metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") -ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) +set_instanceId = "false" +cluster_name = config["clusterName"] + +if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_external_hosts' in config['configurations']['cluster-env']: + ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts'] + set_instanceId = "true" +else: + ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) + has_metric_collector = not len(ams_collector_hosts) == 0 if has_metric_collector: if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_port' in config['configurations']['cluster-env']: - metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] + 'metrics_collector_external_port' in config['configurations']['cluster-env']: + metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port'] else: metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188") if metric_collector_web_address.find(':') != -1: http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py index 44b256e..d9fae76 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py @@ -174,13 +174,19 @@ if stack_supports_storm_kerberos: else: storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.nonsecure.transport'] -ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) +set_instanceId = "false" +if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_external_hosts' in config['configurations']['cluster-env']: + ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts'] + set_instanceId = "true" +else: + ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) has_metric_collector = not len(ams_collector_hosts) == 0 metric_collector_port = None if has_metric_collector: if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_port' in config['configurations']['cluster-env']: - metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] + 'metrics_collector_external_port' in config['configurations']['cluster-env']: + metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port'] else: metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188") if metric_collector_web_address.find(':') != -1: http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 index f2a75cf..51162e8 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 @@ -67,4 +67,7 @@ metrics_collector: truststore.type : "{{metric_truststore_type}}" truststore.password : "{{metric_truststore_password}}" + instanceId={{cluster_name}} + set.instanceId={{set_instanceId}} + {% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 index 001ff73..0501039 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 @@ -23,6 +23,8 @@ zookeeper.quorum={{zookeeper_quorum}} maxRowCacheSize=10000 sendInterval={{metrics_report_interval}}000 clusterReporterAppId=nimbus +instanceId={{cluster_name}} +set.instanceId={{set_instanceId}} # HTTPS properties truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py index f3d1951..2e7f5cd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py @@ -111,7 +111,14 @@ jtnode_host = default("/clusterHostInfo/jtnode_host", []) namenode_host = default("/clusterHostInfo/namenode_host", []) zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) -ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) +cluster_name = config["clusterName"] +set_instanceId = "false" +if 'cluster-env' in config['configurations'] and \ + 'metrics_collector_external_hosts' in config['configurations']['cluster-env']: + ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts'] + set_instanceId = "true" +else: + ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) has_namenode = not len(namenode_host) == 0 has_resourcemanager = not len(rm_host) == 0 @@ -137,8 +144,8 @@ if has_ganglia_server: metric_collector_port = None if has_metric_collector: if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_port' in config['configurations']['cluster-env']: - metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] + 'metrics_collector_external_port' in config['configurations']['cluster-env']: + metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port'] else: metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188") if metric_collector_web_address.find(':') != -1: http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 index 07cb93e..5e1ebfd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 @@ -75,6 +75,8 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue *.sink.timeline.zookeeper.quorum={{zookeeper_quorum}} *.sink.timeline.protocol={{metric_collector_protocol}} *.sink.timeline.port={{metric_collector_port}} +*.sink.timeline.instanceId={{cluster_name}} +*.sink.timeline.set.instanceId={{set_instanceId}} # HTTPS properties *.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml index 4aadb83..fae61d3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml @@ -86,6 +86,8 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue *.sink.timeline.zookeeper.quorum={{zookeeper_quorum}} *.sink.timeline.protocol={{metric_collector_protocol}} *.sink.timeline.port={{metric_collector_port}} +*.sink.timeline.instanceId={{cluster_name}} +*.sink.timeline.set.instanceId={{set_instanceId}} # HTTPS properties *.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/176de64d/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index fa7419f..87a9034 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -603,8 +603,8 @@ "user_group": "hadoop", "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0", "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}", - "metrics_collector_vip_host": "c6402.ambari.apache.org", - "metrics_collector_vip_port": "6189", + "metrics_collector_external_hosts": "c6402.ambari.apache.org", + "metrics_collector_external_port": "6189", "override_uid" : "true", "fetch_nonlocal_groups": "true", "manage_dirs_on_root": "true",
