Repository: ambari Updated Branches: refs/heads/trunk 1e94bc1ab -> 9e052eed1
AMBARI-8566. Remaining changes to Hadoop Sink implementation. (mpapirkovskyy) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9e052eed Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9e052eed Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9e052eed Branch: refs/heads/trunk Commit: 9e052eed1a582072a448e923f50107e00716c04f Parents: 1e94bc1 Author: Myroslav Papirkovskyy <mpapyrkovs...@hortonworks.com> Authored: Fri Dec 5 21:43:21 2014 +0200 Committer: Myroslav Papirkovskyy <mpapyrkovs...@hortonworks.com> Committed: Sat Dec 6 00:11:06 2014 +0200 ---------------------------------------------------------------------- ambari-metrics/ambari-metrics-common/pom.xml | 29 +---- .../flume/FlumeTimelineMetricsSinkTest.java | 117 +++++++++++++++++++ .../flume/FlumeTimelineMetricsSinkTest.java | 117 ------------------- .../ambari-metrics-hadoop-sink/pom.xml | 47 ++++++-- .../src/main/assemblies/sink-jar.xml | 37 ++++++ .../timeline/HadoopTimelineMetricsSinkTest.java | 115 ++++++++++++++++++ .../conf/unix/ambari-metrics-monitor | 1 + .../ambari-metrics-host-monitoring/pom.xml | 7 +- ambari-metrics/pom.xml | 27 ----- .../templates/hadoop-metrics2.properties.j2 | 4 +- ...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 10 +- ...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 8 +- .../hadoop-metrics2-hbase.properties.j2 | 8 +- pom.xml | 6 + 14 files changed, 333 insertions(+), 200 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-common/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/pom.xml b/ambari-metrics/ambari-metrics-common/pom.xml index 786ad93..7bfd9f1 100644 --- a/ambari-metrics/ambari-metrics-common/pom.xml +++ b/ambari-metrics/ambari-metrics-common/pom.xml @@ -31,33 +31,6 @@ <artifactId>maven-compiler-plugin</artifactId> <version>3.0</version> </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <version>1.8</version> - <executions> - <execution> - <id>parse-version</id> - <phase>validate</phase> - <goals> - <goal>parse-version</goal> - </goals> - </execution> - <execution> - <id>regex-property</id> - <goals> - <goal>regex-property</goal> - </goals> - <configuration> - <name>ambariVersion</name> - <value>${project.version}</value> - <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex> - <replacement>$1.$2.$3</replacement> - <failIfNoMatch>false</failIfNoMatch> - </configuration> - </execution> - </executions> - </plugin> </plugins> </build> @@ -78,4 +51,4 @@ <version>1.8.0</version> </dependency> </dependencies> -</project> \ No newline at end of file +</project> http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java new file mode 100644 index 0000000..0275db6 --- /dev/null +++ b/ambari-metrics/ambari-metrics-flume-sink/src/test/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.sink.flume; + +import org.apache.commons.httpclient.HttpClient; +import org.apache.flume.instrumentation.util.JMXPollUtil; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache; +import org.easymock.EasyMock; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.Collections; + +import static org.powermock.api.easymock.PowerMock.*; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(JMXPollUtil.class) +public class FlumeTimelineMetricsSinkTest { + @Test + public void testNonNumericMetricMetricExclusion() throws InterruptedException { + FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink(); + TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink); + flumeTimelineMetricsSink.setPollFrequency(1); + HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class); + flumeTimelineMetricsSink.setHttpClient(httpClient); + mockStatic(JMXPollUtil.class); + EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn( + Collections.singletonMap("component1", Collections.singletonMap("key1", "value1"))).once(); + replay(JMXPollUtil.class, timelineMetricsCache, httpClient); + flumeTimelineMetricsSink.start(); + Thread.sleep(5); + flumeTimelineMetricsSink.stop(); + verifyAll(); + } + + @Test + public void testNumericMetricMetricSubmission() throws InterruptedException { + FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink(); + TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink); + flumeTimelineMetricsSink.setPollFrequency(1); + HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class); + flumeTimelineMetricsSink.setHttpClient(httpClient); + mockStatic(JMXPollUtil.class); + EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn( + Collections.singletonMap("component1", Collections.singletonMap("key1", "42"))).once(); + replay(JMXPollUtil.class, timelineMetricsCache, httpClient); + flumeTimelineMetricsSink.start(); + Thread.sleep(5); + flumeTimelineMetricsSink.stop(); + verifyAll(); + } + + private TimelineMetricsCache getTimelineMetricsCache(FlumeTimelineMetricsSink flumeTimelineMetricsSink) { + TimelineMetricsCache timelineMetricsCache = EasyMock.createNiceMock(TimelineMetricsCache.class); + flumeTimelineMetricsSink.setMetricsCache(timelineMetricsCache); + EasyMock.expect(timelineMetricsCache.getTimelineMetric("key1")) + .andReturn(new TimelineMetric()).once(); + timelineMetricsCache.putTimelineMetric(EasyMock.anyObject(TimelineMetric.class)); + EasyMock.expectLastCall().once(); + return timelineMetricsCache; + } + + @Test + public void testMonitorRestart() throws InterruptedException { + FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink(); + TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink); + flumeTimelineMetricsSink.setPollFrequency(1); + HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class); + flumeTimelineMetricsSink.setHttpClient(httpClient); + mockStatic(JMXPollUtil.class); + EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn( + Collections.singletonMap("component1", Collections.singletonMap("key1", "42"))).once(); + flumeTimelineMetricsSink.start(); + flumeTimelineMetricsSink.stop(); + replay(JMXPollUtil.class, timelineMetricsCache, httpClient); + flumeTimelineMetricsSink.start(); + Thread.sleep(5); + flumeTimelineMetricsSink.stop(); + verifyAll(); + } + + @Test + public void testMetricsRetrievalExceptionTolerance() throws InterruptedException { + FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink(); + TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink); + flumeTimelineMetricsSink.setPollFrequency(1); + HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class); + flumeTimelineMetricsSink.setHttpClient(httpClient); + mockStatic(JMXPollUtil.class); + EasyMock.expect(JMXPollUtil.getAllMBeans()). + andThrow(new RuntimeException("Failed to retrieve Flume Properties")).once(); + replay(JMXPollUtil.class, timelineMetricsCache, httpClient); + flumeTimelineMetricsSink.start(); + Thread.sleep(5); + flumeTimelineMetricsSink.stop(); + verifyAll(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-flume-sink/src/test/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-flume-sink/src/test/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-flume-sink/src/test/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java deleted file mode 100644 index 0275db6..0000000 --- a/ambari-metrics/ambari-metrics-flume-sink/src/test/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSinkTest.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metrics2.sink.flume; - -import org.apache.commons.httpclient.HttpClient; -import org.apache.flume.instrumentation.util.JMXPollUtil; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; -import org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache; -import org.easymock.EasyMock; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.Collections; - -import static org.powermock.api.easymock.PowerMock.*; - -@RunWith(PowerMockRunner.class) -@PrepareForTest(JMXPollUtil.class) -public class FlumeTimelineMetricsSinkTest { - @Test - public void testNonNumericMetricMetricExclusion() throws InterruptedException { - FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink(); - TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink); - flumeTimelineMetricsSink.setPollFrequency(1); - HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class); - flumeTimelineMetricsSink.setHttpClient(httpClient); - mockStatic(JMXPollUtil.class); - EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn( - Collections.singletonMap("component1", Collections.singletonMap("key1", "value1"))).once(); - replay(JMXPollUtil.class, timelineMetricsCache, httpClient); - flumeTimelineMetricsSink.start(); - Thread.sleep(5); - flumeTimelineMetricsSink.stop(); - verifyAll(); - } - - @Test - public void testNumericMetricMetricSubmission() throws InterruptedException { - FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink(); - TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink); - flumeTimelineMetricsSink.setPollFrequency(1); - HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class); - flumeTimelineMetricsSink.setHttpClient(httpClient); - mockStatic(JMXPollUtil.class); - EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn( - Collections.singletonMap("component1", Collections.singletonMap("key1", "42"))).once(); - replay(JMXPollUtil.class, timelineMetricsCache, httpClient); - flumeTimelineMetricsSink.start(); - Thread.sleep(5); - flumeTimelineMetricsSink.stop(); - verifyAll(); - } - - private TimelineMetricsCache getTimelineMetricsCache(FlumeTimelineMetricsSink flumeTimelineMetricsSink) { - TimelineMetricsCache timelineMetricsCache = EasyMock.createNiceMock(TimelineMetricsCache.class); - flumeTimelineMetricsSink.setMetricsCache(timelineMetricsCache); - EasyMock.expect(timelineMetricsCache.getTimelineMetric("key1")) - .andReturn(new TimelineMetric()).once(); - timelineMetricsCache.putTimelineMetric(EasyMock.anyObject(TimelineMetric.class)); - EasyMock.expectLastCall().once(); - return timelineMetricsCache; - } - - @Test - public void testMonitorRestart() throws InterruptedException { - FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink(); - TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink); - flumeTimelineMetricsSink.setPollFrequency(1); - HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class); - flumeTimelineMetricsSink.setHttpClient(httpClient); - mockStatic(JMXPollUtil.class); - EasyMock.expect(JMXPollUtil.getAllMBeans()).andReturn( - Collections.singletonMap("component1", Collections.singletonMap("key1", "42"))).once(); - flumeTimelineMetricsSink.start(); - flumeTimelineMetricsSink.stop(); - replay(JMXPollUtil.class, timelineMetricsCache, httpClient); - flumeTimelineMetricsSink.start(); - Thread.sleep(5); - flumeTimelineMetricsSink.stop(); - verifyAll(); - } - - @Test - public void testMetricsRetrievalExceptionTolerance() throws InterruptedException { - FlumeTimelineMetricsSink flumeTimelineMetricsSink = new FlumeTimelineMetricsSink(); - TimelineMetricsCache timelineMetricsCache = getTimelineMetricsCache(flumeTimelineMetricsSink); - flumeTimelineMetricsSink.setPollFrequency(1); - HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class); - flumeTimelineMetricsSink.setHttpClient(httpClient); - mockStatic(JMXPollUtil.class); - EasyMock.expect(JMXPollUtil.getAllMBeans()). - andThrow(new RuntimeException("Failed to retrieve Flume Properties")).once(); - replay(JMXPollUtil.class, timelineMetricsCache, httpClient); - flumeTimelineMetricsSink.start(); - Thread.sleep(5); - flumeTimelineMetricsSink.stop(); - verifyAll(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml b/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml index 1e854e2..76f9fce 100644 --- a/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml +++ b/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml @@ -32,20 +32,36 @@ limitations under the License. <plugins> <plugin> <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <descriptors> - <descriptor>src/main/assemblies/sink.xml</descriptor> - </descriptors> - <tarLongFileMode>gnu</tarLongFileMode> - </configuration> <executions> <execution> + <configuration> + <descriptors> + <descriptor>src/main/assemblies/sink.xml</descriptor> + </descriptors> + <tarLongFileMode>gnu</tarLongFileMode> + </configuration> <id>build-tarball</id> <phase>package</phase> <goals> <goal>single</goal> </goals> </execution> + + <execution> + <configuration> + <descriptors> + <descriptor>src/main/assemblies/sink-jar.xml</descriptor> + </descriptors> + <tarLongFileMode>gnu</tarLongFileMode> + <appendAssemblyId>false</appendAssemblyId> + <finalName>${project.artifactId}-with-common-${project.version}</finalName> + </configuration> + <id>build-jar</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> </executions> </plugin> <plugin> @@ -115,11 +131,11 @@ limitations under the License. <groupname>root</groupname> <sources> <source> - <location>target/${project.artifactId}-${project.version}.jar</location> + <location>target/${project.artifactId}-with-common-${project.version}.jar</location> </source> <softlinkSource> <destination>ambari-metrics-hadoop-sink.jar</destination> - <location>/usr/lib/ambari-metrics-hadoop-sink/${project.artifactId}-${project.version}.jar</location> + <location>/usr/lib/ambari-metrics-hadoop-sink/${project.artifactId}-with-common-${project.version}.jar</location> </softlinkSource> </sources> @@ -135,7 +151,7 @@ limitations under the License. <dependency> <groupId>org.apache.ambari</groupId> <artifactId>ambari-metrics-common</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> @@ -179,9 +195,16 @@ limitations under the License. <scope>compile</scope> </dependency> <dependency> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-core-asl</artifactId> - <version>1.8.0</version> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + <version>4.10</version> + </dependency> + <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <version>3.2</version> + <scope>test</scope> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink-jar.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink-jar.xml b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink-jar.xml new file mode 100644 index 0000000..83c3099 --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink-jar.xml @@ -0,0 +1,37 @@ +<?xml version='1.0'?> + +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<assembly> + <id>hadoop-sink-jar</id> + <formats> + <format>jar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <dependencySets> + <dependencySet> + <outputDirectory>/</outputDirectory> + <unpack>true</unpack> + <includes> + <include>org.apache.ambari:ambari-metrics-common</include> + <include>org.apache.ambari:ambari-metrics-hadoop-sink</include> + </includes> + </dependencySet> + </dependencySets> +</assembly> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java new file mode 100644 index 0000000..2421a73 --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.sink.timeline; + +import org.apache.commons.configuration.SubsetConfiguration; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.sink.timeline.base.AbstractTimelineMetricsSink; +import org.easymock.IAnswer; + +import java.util.Arrays; +import java.util.Iterator; + +import static org.apache.hadoop.metrics2.sink.timeline.base.AbstractTimelineMetricsSink.*; +import static org.easymock.EasyMock.anyInt; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +public class HadoopTimelineMetricsSinkTest { + + @org.junit.Test + public void testPutMetrics() throws Exception { + HadoopTimelineMetricsSink sink = new HadoopTimelineMetricsSink(); + + SubsetConfiguration conf = createNiceMock(SubsetConfiguration.class); + expect(conf.getString(eq("slave.host.name"))).andReturn("testhost").anyTimes(); + expect(conf.getParent()).andReturn(null).anyTimes(); + expect(conf.getPrefix()).andReturn("service").anyTimes(); + expect(conf.getString(eq(COLLECTOR_HOST_PROPERTY))).andReturn("localhost:63188").anyTimes(); + + expect(conf.getInt(eq(MAX_METRIC_ROW_CACHE_SIZE), anyInt())).andReturn(10).anyTimes(); + expect(conf.getInt(eq(METRICS_SEND_INTERVAL), anyInt())).andReturn(1000).anyTimes(); + + conf.setListDelimiter(eq(',')); + expectLastCall().anyTimes(); + + expect(conf.getKeys()).andReturn(new Iterator() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public Object next() { + return null; + } + + @Override + public void remove() { + + } + }).once(); + + + HttpClient httpClient = createNiceMock(HttpClient.class); + + expect(httpClient.executeMethod(anyObject(PostMethod.class))).andReturn(200).once(); //metrics send only once due to caching + + AbstractMetric metric = createNiceMock(AbstractMetric.class); + expect(metric.name()).andReturn("metricName").anyTimes(); + expect(metric.value()).andReturn(9.5687).anyTimes(); + //TODO currently only numeric metrics are supported + + MetricsRecord record = createNiceMock(MetricsRecord.class); + expect(record.name()).andReturn("testName").anyTimes(); + expect(record.context()).andReturn("testContext").anyTimes(); + expect(record.timestamp()).andAnswer(new IAnswer<Long>() { + @Override + public Long answer() throws Throwable { + return System.currentTimeMillis(); + } + }).anyTimes(); + + expect(record.metrics()).andReturn(Arrays.asList(metric)).anyTimes(); + + + replay(conf, httpClient, record, metric); + + sink.setHttpClient(httpClient); + sink.init(conf); + + sink.putMetrics(record); + + Thread.sleep(1500L); + + sink.putMetrics(record); + + verify(conf, httpClient, record, metric); + + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor b/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor index fd086d9..9198ce9 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor +++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor @@ -67,6 +67,7 @@ function write_pidfile fi } +#TODO decide if rebuild on each start (pretty quickly) to tolerate major node changes (like kernel update) #build psutil if [ ! "$(ls -A ${RESOURCE_MONITORING_DIR}/psutil/build)" ]; then echo "Building psutil..." http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/ambari-metrics-host-monitoring/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/pom.xml b/ambari-metrics/ambari-metrics-host-monitoring/pom.xml index b354461..ce396df 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/pom.xml +++ b/ambari-metrics/ambari-metrics-host-monitoring/pom.xml @@ -120,6 +120,7 @@ <autoRequires>false</autoRequires> <requires> <require>ambari-metrics-hadoop-sink</require> + <require>ambari-metrics-flume-sink</require> <require>${python.ver}</require> <require>gcc</require> <require>python-devel</require> @@ -172,6 +173,10 @@ <location> ${project.basedir}/src/main/python/psutil </location> + <excludes> + <exclude>build/**</exclude> + <exclude>build/*</exclude> + </excludes> </source> </sources> </mapping> @@ -208,7 +213,7 @@ <executions> <execution> <id>psutils-compile</id> - <phase>process-classes</phase> + <phase>process-test-classes</phase> <goals> <goal>run</goal> </goals> http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-metrics/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml index b090205..12c571b 100644 --- a/ambari-metrics/pom.xml +++ b/ambari-metrics/pom.xml @@ -67,33 +67,6 @@ </executions> </plugin> <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <version>1.8</version> - <executions> - <execution> - <id>parse-version</id> - <phase>validate</phase> - <goals> - <goal>parse-version</goal> - </goals> - </execution> - <execution> - <id>regex-property</id> - <goals> - <goal>regex-property</goal> - </goals> - <configuration> - <name>ambariVersion</name> - <value>${project.version}</value> - <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex> - <replacement>$1.$2.$3</replacement> - <failIfNoMatch>false</failIfNoMatch> - </configuration> - </execution> - </executions> - </plugin> - <plugin> <artifactId>maven-compiler-plugin</artifactId> <version>3.0</version> </plugin> http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 index 31ca5c3..d7a66ab 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 @@ -67,7 +67,7 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue {% if has_metric_collector %} *.period=60 -*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink *.sink.timeline.period=10 datanode.sink.timeline.collector={{metric_collector_host}}:8188 namenode.sink.timeline.collector={{metric_collector_host}}:8188 @@ -80,4 +80,4 @@ supervisor.sink.timeline.collector={{metric_collector_host}}:8188 maptask.sink.timeline.collector={{metric_collector_host}}:8188 reducetask.sink.timeline.collector={{metric_collector_host}}:8188 -{% endif %} \ No newline at end of file +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 index fc033a7..2f13b28 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 @@ -86,20 +86,20 @@ hbase.sink.ganglia.servers={{ganglia_server_host}}:8663 # If this variable is left out, then the default is no expiration. hbase.extendedperiod = 3600 -hbase.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.period=10 hbase.collector={{metric_collector_host}}:8188 -jvm.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink jvm.period=10 jvm.collector={{metric_collector_host}}:8188 -rpc.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink rpc.period=10 rpc.collector={{metric_collector_host}}:8188 -hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.sink.timeline.period=10 hbase.sink.timeline.collector={{metric_collector_host}}:8188 -{% endif %} \ No newline at end of file +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 index 97e76d8..1ca3a98 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 @@ -85,19 +85,19 @@ hbase.sink.ganglia.servers={{ganglia_server_host}}:8656 # If this variable is left out, then the default is no expiration. hbase.extendedperiod = 3600 -hbase.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.period=10 hbase.collector={{metric_collector_host}}:8188 -jvm.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink jvm.period=10 jvm.collector={{metric_collector_host}}:8188 -rpc.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink rpc.period=10 rpc.collector={{metric_collector_host}}:8188 -hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.sink.timeline.period=10 hbase.sink.timeline.collector={{metric_collector_host}}:8188 http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hadoop-metrics2-hbase.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hadoop-metrics2-hbase.properties.j2 index 8dc4bea..0d3ce44 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hadoop-metrics2-hbase.properties.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/hadoop-metrics2-hbase.properties.j2 @@ -38,18 +38,18 @@ # If this variable is left out, then the default is no expiration. hbase.extendedperiod = 3600 -hbase.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.period=10 hbase.collector={{ams_collector_host_single}}:8188 -jvm.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink jvm.period=10 jvm.collector={{ams_collector_host_single}}:8188 -rpc.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink rpc.period=10 rpc.collector={{ams_collector_host_single}}:8188 -hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.sink.timeline.period=10 hbase.sink.timeline.collector={{ams_collector_host_single}}:8188 http://git-wip-us.apache.org/repos/asf/ambari/blob/9e052eed/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 87fb4b8..79f951b 100644 --- a/pom.xml +++ b/pom.xml @@ -361,6 +361,12 @@ </modules> </profile> <profile> + <id>ambari-metrics</id> + <modules> + <module>ambari-metrics</module> + </modules> + </profile> + <profile> <id>windows</id> <activation> <os>
