AMBARI-5707. Replace Ganglia with high performant and pluggable Metrics System. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a52f8a55 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a52f8a55 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a52f8a55 Branch: refs/heads/trunk Commit: a52f8a5572f689351a22d41f62bd083c27a3bfa8 Parents: c6f5743 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Mon Dec 1 14:49:42 2014 -0800 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Tue Dec 2 09:26:41 2014 -0800 ---------------------------------------------------------------------- .gitignore | 1 + .../ambari-metrics-hadoop-sink/pom.xml | 188 + .../src/main/assemblies/empty.xml | 21 + .../src/main/assemblies/sink.xml | 34 + .../conf/hadoop-metrics2-hbase.properties.j2 | 49 + .../src/main/conf/hadoop-metrics2.properties.j2 | 58 + .../timeline/AbstractTimelineMetricsSink.java | 101 + .../metrics2/sink/timeline/TimelineMetric.java | 172 + .../metrics2/sink/timeline/TimelineMetrics.java | 102 + .../sink/timeline/TimelineMetricsCache.java | 128 + .../sink/timeline/TimelineMetricsSink.java | 211 ++ .../conf/unix/ambari-metrics-monitor | 203 ++ .../ambari-metrics-host-monitoring/pom.xml | 273 ++ .../src/main/package/rpm/preremove.sh | 28 + .../src/main/python/__init__.py | 21 + .../src/main/python/core/__init__.py | 33 + .../main/python/core/application_metric_map.py | 130 + .../src/main/python/core/config_reader.py | 127 + .../src/main/python/core/controller.py | 103 + .../src/main/python/core/emitter.py | 88 + .../src/main/python/core/event_definition.py | 85 + .../src/main/python/core/host_info.py | 190 + .../src/main/python/core/metric_collector.py | 87 + .../src/main/python/main.py | 64 + .../src/main/python/psutil/LICENSE | 27 + .../src/main/python/psutil/MANIFEST.in | 14 + .../src/main/python/psutil/Makefile | 77 + .../src/main/python/psutil/README | 270 ++ .../src/main/python/psutil/build.py | 57 + .../src/main/python/psutil/docs/Makefile | 177 + .../src/main/python/psutil/docs/README | 15 + .../python/psutil/docs/_static/copybutton.js | 57 + .../main/python/psutil/docs/_static/sidebar.js | 161 + .../python/psutil/docs/_template/globaltoc.html | 12 + .../psutil/docs/_template/indexcontent.html | 4 + .../psutil/docs/_template/indexsidebar.html | 16 + .../main/python/psutil/docs/_template/page.html | 66 + .../_themes/pydoctheme/static/pydoctheme.css | 187 + .../psutil/docs/_themes/pydoctheme/theme.conf | 23 + .../src/main/python/psutil/docs/conf.py | 253 ++ .../src/main/python/psutil/docs/index.rst | 1247 +++++++ .../src/main/python/psutil/docs/make.bat | 242 ++ .../main/python/psutil/examples/disk_usage.py | 63 + .../src/main/python/psutil/examples/free.py | 42 + .../src/main/python/psutil/examples/iotop.py | 178 + .../src/main/python/psutil/examples/killall.py | 32 + .../src/main/python/psutil/examples/meminfo.py | 69 + .../src/main/python/psutil/examples/netstat.py | 65 + .../src/main/python/psutil/examples/nettop.py | 165 + .../src/main/python/psutil/examples/pmap.py | 58 + .../python/psutil/examples/process_detail.py | 162 + .../src/main/python/psutil/examples/top.py | 232 ++ .../src/main/python/psutil/examples/who.py | 34 + .../src/main/python/psutil/make.bat | 176 + .../src/main/python/psutil/psutil/__init__.py | 1987 +++++++++++ .../src/main/python/psutil/psutil/_common.py | 258 ++ .../src/main/python/psutil/psutil/_compat.py | 433 +++ .../src/main/python/psutil/psutil/_psbsd.py | 389 +++ .../src/main/python/psutil/psutil/_pslinux.py | 1225 +++++++ .../src/main/python/psutil/psutil/_psosx.py | 341 ++ .../src/main/python/psutil/psutil/_psposix.py | 157 + .../src/main/python/psutil/psutil/_pssunos.py | 533 +++ .../src/main/python/psutil/psutil/_psutil_bsd.c | 2212 ++++++++++++ .../src/main/python/psutil/psutil/_psutil_bsd.h | 51 + .../main/python/psutil/psutil/_psutil_common.c | 37 + .../main/python/psutil/psutil/_psutil_common.h | 10 + .../main/python/psutil/psutil/_psutil_linux.c | 510 +++ .../main/python/psutil/psutil/_psutil_linux.h | 20 + .../src/main/python/psutil/psutil/_psutil_osx.c | 1881 ++++++++++ .../src/main/python/psutil/psutil/_psutil_osx.h | 41 + .../main/python/psutil/psutil/_psutil_posix.c | 128 + .../main/python/psutil/psutil/_psutil_posix.h | 10 + .../main/python/psutil/psutil/_psutil_sunos.c | 1290 +++++++ .../main/python/psutil/psutil/_psutil_sunos.h | 27 + .../main/python/psutil/psutil/_psutil_windows.c | 3241 ++++++++++++++++++ .../main/python/psutil/psutil/_psutil_windows.h | 70 + .../src/main/python/psutil/psutil/_pswindows.py | 485 +++ .../psutil/psutil/arch/bsd/process_info.c | 285 ++ .../psutil/psutil/arch/bsd/process_info.h | 15 + .../psutil/psutil/arch/osx/process_info.c | 293 ++ .../psutil/psutil/arch/osx/process_info.h | 16 + .../python/psutil/psutil/arch/windows/glpi.h | 41 + .../psutil/psutil/arch/windows/ntextapi.h | 287 ++ .../psutil/arch/windows/process_handles.c | 336 ++ .../psutil/arch/windows/process_handles.h | 10 + .../psutil/psutil/arch/windows/process_info.c | 443 +++ .../psutil/psutil/arch/windows/process_info.h | 17 + .../psutil/psutil/arch/windows/security.c | 238 ++ .../psutil/psutil/arch/windows/security.h | 17 + .../src/main/python/psutil/setup.py | 198 ++ .../python/core/TestApplicationMetricMap.py | 67 + .../src/test/python/core/TestEmitter.py | 78 + .../src/test/python/core/TestHostInfo.py | 97 + .../src/test/python/core/TestMetricCollector.py | 49 + .../src/test/python/unitTests.py | 133 + .../conf/unix/ambari-metrics-collector | 269 ++ .../conf/unix/ams-env.sh | 16 + .../conf/unix/ams-site.xml | 25 + .../conf/unix/log4j.properties | 31 + .../ambari-metrics-timelineservice/pom.xml | 593 ++++ .../src/main/assemblies/ats.xml | 34 + .../src/main/assemblies/empty.xml | 21 + .../src/main/assemblies/phoenix-client.xml | 62 + .../phoenix-components-major-client.xml | 53 + .../assemblies/phoenix-components-minimal.xml | 71 + .../src/main/assemblies/phoenix-server.xml | 46 + .../src/main/assemblies/simulator.xml | 68 + .../main/conf/hbase-site-metrics-service.xml | 80 + .../src/main/conf/simulator-log4j.xml | 45 + .../ApplicationHistoryClientService.java | 211 ++ .../ApplicationHistoryManager.java | 28 + .../ApplicationHistoryManagerImpl.java | 250 ++ .../ApplicationHistoryReader.java | 117 + .../ApplicationHistoryServer.java | 197 ++ .../ApplicationHistoryStore.java | 37 + .../ApplicationHistoryWriter.java | 112 + .../FileSystemApplicationHistoryStore.java | 784 +++++ .../MemoryApplicationHistoryStore.java | 274 ++ .../NullApplicationHistoryStore.java | 127 + .../metrics/loadsimulator/LoadRunner.java | 160 + .../loadsimulator/MetricsLoadSimulator.java | 120 + .../loadsimulator/MetricsSenderWorker.java | 65 + .../metrics/loadsimulator/data/AppID.java | 42 + .../metrics/loadsimulator/data/AppMetrics.java | 48 + .../loadsimulator/data/ApplicationInstance.java | 59 + .../data/HostMetricsGenerator.java | 63 + .../metrics/loadsimulator/data/Metric.java | 71 + .../data/MetricsGeneratorConfigurer.java | 95 + .../loadsimulator/net/MetricsSender.java | 31 + .../loadsimulator/net/RestMetricsSender.java | 95 + .../loadsimulator/net/StdOutMetricsSender.java | 56 + .../metrics/loadsimulator/net/UrlService.java | 101 + .../metrics/loadsimulator/util/Json.java | 63 + .../util/RandomMetricsProvider.java | 40 + .../loadsimulator/util/TimeStampProvider.java | 52 + .../timeline/AbstractTimelineAggregator.java | 229 ++ .../metrics/timeline/Aggregator.java | 60 + .../metrics/timeline/ConnectionProvider.java | 30 + .../timeline/DefaultPhoenixDataSource.java | 78 + .../timeline/HBaseTimelineMetricStore.java | 192 ++ .../metrics/timeline/MetricAggregate.java | 110 + .../timeline/MetricClusterAggregate.java | 74 + .../metrics/timeline/MetricHostAggregate.java | 81 + .../MetricsInitializationException.java | 42 + .../metrics/timeline/PhoenixHBaseAccessor.java | 678 ++++ .../metrics/timeline/PhoenixTransactSQL.java | 528 +++ .../metrics/timeline/TimelineClusterMetric.java | 97 + .../timeline/TimelineMetricAggregator.java | 144 + .../TimelineMetricAggregatorFactory.java | 99 + .../TimelineMetricClusterAggregator.java | 220 ++ .../TimelineMetricClusterAggregatorHourly.java | 173 + .../timeline/TimelineMetricConfiguration.java | 115 + .../metrics/timeline/TimelineMetricStore.java | 70 + .../records/ApplicationAttemptFinishData.java | 95 + .../records/ApplicationAttemptHistoryData.java | 171 + .../records/ApplicationAttemptStartData.java | 82 + .../records/ApplicationFinishData.java | 94 + .../records/ApplicationHistoryData.java | 213 ++ .../records/ApplicationStartData.java | 106 + .../records/ContainerFinishData.java | 90 + .../records/ContainerHistoryData.java | 182 + .../records/ContainerStartData.java | 92 + .../pb/ApplicationAttemptFinishDataPBImpl.java | 239 ++ .../pb/ApplicationAttemptStartDataPBImpl.java | 208 ++ .../impl/pb/ApplicationFinishDataPBImpl.java | 226 ++ .../impl/pb/ApplicationStartDataPBImpl.java | 229 ++ .../impl/pb/ContainerFinishDataPBImpl.java | 204 ++ .../impl/pb/ContainerStartDataPBImpl.java | 258 ++ .../timeline/EntityIdentifier.java | 100 + .../timeline/GenericObjectMapper.java | 135 + .../timeline/LeveldbTimelineStore.java | 1473 ++++++++ .../timeline/MemoryTimelineStore.java | 360 ++ .../timeline/NameValuePair.java | 59 + .../timeline/TimelineReader.java | 155 + .../timeline/TimelineStore.java | 29 + .../timeline/TimelineWriter.java | 46 + .../timeline/package-info.java | 20 + .../webapp/AHSController.java | 55 + .../webapp/AHSLogsPage.java | 55 + .../webapp/AHSView.java | 90 + .../webapp/AHSWebApp.java | 63 + .../webapp/AHSWebServices.java | 162 + .../webapp/AppAttemptPage.java | 69 + .../webapp/AppPage.java | 71 + .../webapp/ContainerPage.java | 41 + .../webapp/JAXBContextResolver.java | 64 + .../webapp/NavBlock.java | 51 + .../webapp/TimelineWebServices.java | 504 +++ .../src/main/resources/metrics_def/DATANODE.dat | 101 + .../src/main/resources/metrics_def/HOST.dat | 23 + .../main/resources/metrics_def/MASTER_HBASE.dat | 69 + .../src/main/resources/metrics_def/NAMENODE.dat | 205 ++ .../main/resources/metrics_def/NODEMANAGER.dat | 67 + .../resources/metrics_def/RESOURCEMANAGER.dat | 88 + .../main/resources/metrics_def/SLAVE_HBASE.dat | 178 + .../src/main/resources/scripts/ams_query.py | 209 ++ .../src/main/resources/scripts/start.sh | 30 + .../src/main/resources/scripts/start_slaves.sh | 27 + .../src/main/resources/scripts/status_slaves.sh | 22 + .../src/main/resources/scripts/stop.sh | 32 + .../src/main/resources/scripts/stop_slaves.sh | 26 + .../src/test/conf/ams-site.xml | 29 + .../src/test/conf/hadoop-policy.xml | 134 + .../src/test/conf/hbase-site.xml | 230 ++ .../ApplicationHistoryStoreTestUtils.java | 84 + .../TestApplicationHistoryClientService.java | 209 ++ .../TestApplicationHistoryManagerImpl.java | 76 + .../TestApplicationHistoryServer.java | 217 ++ .../TestFileSystemApplicationHistoryStore.java | 233 ++ .../TestMemoryApplicationHistoryStore.java | 204 ++ .../loadsimulator/data/TestAppMetrics.java | 134 + .../metrics/loadsimulator/data/TestMetric.java | 80 + .../net/TestRestMetricsSender.java | 76 + .../net/TestStdOutMetricsSender.java | 37 + .../util/TestRandomMetricsProvider.java | 36 + .../util/TestTimeStampProvider.java | 51 + .../timeline/AbstractMiniHBaseClusterTest.java | 113 + .../AbstractPhoenixConnectionlessTest.java | 111 + .../metrics/timeline/ITClusterAggregator.java | 384 +++ .../metrics/timeline/ITMetricAggregator.java | 310 ++ .../metrics/timeline/TestClusterSuite.java | 30 + .../timeline/TestMetricHostAggregate.java | 65 + .../timeline/TestPhoenixTransactSQL.java | 43 + .../timeline/TestTimelineMetricStore.java | 81 + .../timeline/TestGenericObjectMapper.java | 102 + .../timeline/TestLeveldbTimelineStore.java | 253 ++ .../timeline/TestMemoryTimelineStore.java | 83 + .../timeline/TimelineStoreTestUtils.java | 789 +++++ .../webapp/TestAHSWebApp.java | 183 + .../webapp/TestAHSWebServices.java | 303 ++ .../webapp/TestTimelineWebServices.java | 391 +++ .../src/test/resources/hbase-default.xml | 36 + .../4.2.0/phoenix-core-tests-4.2.0.jar | Bin 0 -> 992828 bytes .../4.2.0/phoenix-core-tests-4.2.0.pom | 9 + .../phoenix-core-tests/maven-metadata-local.xml | 12 + ambari-metrics/pom.xml | 180 + .../java/org/apache/ambari/server/Role.java | 2 + .../GangliaComponentPropertyProvider.java | 71 - .../GangliaHostComponentPropertyProvider.java | 72 - .../ganglia/GangliaHostPropertyProvider.java | 85 - .../controller/ganglia/GangliaHostProvider.java | 59 - .../controller/ganglia/GangliaMetric.java | 257 -- .../ganglia/GangliaPropertyProvider.java | 676 ---- .../ganglia/GangliaReportPropertyProvider.java | 234 -- .../internal/AbstractProviderModule.java | 356 +- .../internal/StackDefinedPropertyProvider.java | 107 +- .../controller/jmx/JMXPropertyProvider.java | 38 +- .../controller/metrics/MetricHostProvider.java | 78 + .../controller/metrics/MetricsHostProvider.java | 38 - .../metrics/MetricsPropertyProvider.java | 204 ++ .../controller/metrics/MetricsProvider.java | 302 -- .../metrics/MetricsReportPropertyProvider.java | 88 + .../metrics/RestMetricsPropertyProvider.java | 32 +- .../ThreadPoolEnabledPropertyProvider.java | 301 ++ .../GangliaComponentPropertyProvider.java | 73 + .../GangliaHostComponentPropertyProvider.java | 71 + .../ganglia/GangliaHostPropertyProvider.java | 85 + .../metrics/ganglia/GangliaMetric.java | 257 ++ .../ganglia/GangliaPropertyProvider.java | 618 ++++ .../ganglia/GangliaReportPropertyProvider.java | 227 ++ .../timeline/AMSComponentPropertyProvider.java | 55 + .../AMSHostComponentPropertyProvider.java | 56 + .../timeline/AMSHostPropertyProvider.java | 50 + .../metrics/timeline/AMSPropertyProvider.java | 419 +++ .../timeline/AMSReportPropertyProvider.java | 49 + .../controller/utilities/PropertyHelper.java | 2 +- .../org/apache/ambari/server/state/Service.java | 3 +- .../metrics2/sink/timeline/TimelineMetric.java | 184 + .../metrics2/sink/timeline/TimelineMetrics.java | 103 + .../src/main/resources/ganglia_properties.json | 4 +- .../2.0.6/hooks/before-START/scripts/params.py | 5 + .../templates/hadoop-metrics2.properties.j2 | 18 + .../services/HBASE/package/scripts/params.py | 5 + ...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 24 + ...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 24 + .../2.2/services/AMS/configuration/ams-env.xml | 41 + .../AMS/configuration/ams-hbase-env.xml | 137 + .../AMS/configuration/ams-hbase-log4j.xml | 143 + .../AMS/configuration/ams-hbase-policy.xml | 53 + .../AMS/configuration/ams-hbase-site.xml | 252 ++ .../services/AMS/configuration/ams-log4j.xml | 59 + .../2.2/services/AMS/configuration/ams-site.xml | 221 ++ .../stacks/HDP/2.2/services/AMS/metainfo.xml | 104 + .../AMS/package/files/hbaseSmokeVerify.sh | 34 + .../services/AMS/package/scripts/__init__.py | 19 + .../HDP/2.2/services/AMS/package/scripts/ams.py | 89 + .../services/AMS/package/scripts/ams_service.py | 72 + .../services/AMS/package/scripts/functions.py | 40 + .../2.2/services/AMS/package/scripts/hbase.py | 123 + .../AMS/package/scripts/hbase_master.py | 70 + .../AMS/package/scripts/hbase_regionserver.py | 66 + .../AMS/package/scripts/hbase_service.py | 51 + .../AMS/package/scripts/metric_collector.py | 63 + .../AMS/package/scripts/metric_monitor.py | 61 + .../2.2/services/AMS/package/scripts/params.py | 174 + .../AMS/package/scripts/service_check.py | 19 + .../AMS/package/scripts/status_params.py | 30 + .../hadoop-metrics2-hbase.properties.j2 | 55 + .../package/templates/hbase_client_jaas.conf.j2 | 23 + .../templates/hbase_grant_permissions.j2 | 39 + .../package/templates/hbase_master_jaas.conf.j2 | 26 + .../templates/hbase_regionserver_jaas.conf.j2 | 26 + .../AMS/package/templates/metric_groups.conf.j2 | 37 + .../AMS/package/templates/metric_monitor.ini.j2 | 30 + .../AMS/package/templates/regionservers.j2 | 20 + .../services/HBASE/configuration/hbase-env.xml | 139 + .../services/HDFS/configuration/hadoop-env.xml | 4 + .../controller/ganglia/GangliaHelperTest.java | 50 - .../controller/ganglia/GangliaMetricTest.java | 116 - .../ganglia/GangliaPropertyProviderTest.java | 875 ----- .../GangliaReportPropertyProviderTest.java | 123 - .../controller/ganglia/TestStreamProvider.java | 56 - .../internal/AbstractPropertyProviderTest.java | 6 +- .../StackDefinedPropertyProviderTest.java | 86 +- .../metrics/JMXPropertyProviderTest.java | 36 +- .../RestMetricsPropertyProviderTest.java | 22 +- .../metrics/ganglia/GangliaMetricTest.java | 118 + .../ganglia/GangliaPropertyProviderTest.java | 885 +++++ .../GangliaReportPropertyProviderTest.java | 136 + .../metrics/ganglia/TestStreamProvider.java | 56 + .../timeline/AMSPropertyProviderTest.java | 206 ++ .../utilities/PropertyHelperTest.java | 4 +- .../ams/multiple_component_metrics.json | 1274 +++++++ .../resources/ams/multiple_host_metrics.json | 215 ++ .../resources/ams/single_component_metrics.json | 250 ++ .../test/resources/ams/single_host_metric.json | 121 + pom.xml | 5 + 327 files changed, 53832 insertions(+), 3302 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 0430303..a8c4aab 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ derby.log pass.txt ambari-agent/src/test/python/ambari_agent/dummy_files/current-stack velocity.log* +ambari-metrics/ambari-metrics-host-monitoring/src/main/python/psutil/build/ http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml b/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml new file mode 100644 index 0000000..0397e2e --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/pom.xml @@ -0,0 +1,188 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>ambari-metrics</artifactId> + <groupId>org.apache.ambari</groupId> + <version>0.1.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>ambari-metrics-hadoop-sink</artifactId> + <version>0.1.0-SNAPSHOT</version> + <packaging>jar</packaging> + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptors> + <descriptor>src/main/assemblies/sink.xml</descriptor> + </descriptors> + <tarLongFileMode>gnu</tarLongFileMode> + </configuration> + <executions> + <execution> + <id>build-tarball</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.0</version> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.8</version> + <executions> + <execution> + <id>parse-version</id> + <phase>validate</phase> + <goals> + <goal>parse-version</goal> + </goals> + </execution> + <execution> + <id>regex-property</id> + <goals> + <goal>regex-property</goal> + </goals> + <configuration> + <name>ambariVersion</name> + <value>${project.version}</value> + <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex> + <replacement>$1.$2.$3</replacement> + <failIfNoMatch>false</failIfNoMatch> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.github.goldin</groupId> + <artifactId>copy-maven-plugin</artifactId> + <version>0.2.5</version> + <executions> + <execution> + <id>create-archive</id> + <phase>none</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>rpm-maven-plugin</artifactId> + <version>2.0.1</version> + <executions> + <execution> + <phase>none</phase> + <goals> + <goal>rpm</goal> + </goals> + </execution> + </executions> + <configuration> + <name>ambari-metrics-hadoop-sink</name> + <copyright>2012, Apache Software Foundation</copyright> + <group>Development</group> + <description>Maven Recipe: RPM Package.</description> + <mappings> + <mapping> + <directory>/usr/lib/ambari-metrics-hadoop-sink</directory> + <filemode>644</filemode> + <username>root</username> + <groupname>root</groupname> + <sources> + <source> + <location>target/${project.artifactId}-${project.version}.jar</location> + </source> + <softlinkSource> + <destination>ambari-metrics-hadoop-sink.jar</destination> + <location>/usr/lib/ambari-metrics-hadoop-sink/${project.artifactId}-${project.version}.jar</location> + </softlinkSource> + </sources> + + </mapping> + + </mappings> + </configuration> + </plugin> + </plugins> + </build> + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>2.4.0</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-httpclient</groupId> + <artifactId>commons-httpclient</artifactId> + <version>3.1</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.8</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>2.1</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + <version>3.2.1</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>1.1.1</version> + </dependency> + <dependency> + <groupId>commons-configuration</groupId> + <artifactId>commons-configuration</artifactId> + <version>1.6</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + <version>1.9.9</version> + </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + <version>1.9.13</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/empty.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/empty.xml b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/empty.xml new file mode 100644 index 0000000..35738b1 --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/empty.xml @@ -0,0 +1,21 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>empty</id> + <formats/> +</assembly> http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink.xml b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink.xml new file mode 100644 index 0000000..21a6b36 --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/assemblies/sink.xml @@ -0,0 +1,34 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <!--This 'all' id is not appended to the produced bundle because we do this: + http://maven.apache.org/plugins/maven-assembly-plugin/faq.html#required-classifiers + --> + <id>dist</id> + <formats> + <format>dir</format> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <files> + <file> + <source>${project.build.directory}/${artifact.artifactId}-${artifact.version}.jar</source> + <outputDirectory>ambari-metrics-${project.version}/lib/ambari-metrics</outputDirectory> + </file> + </files> +</assembly> http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2-hbase.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2-hbase.properties.j2 b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2-hbase.properties.j2 new file mode 100644 index 0000000..6e64421 --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2-hbase.properties.j2 @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://wiki.apache.org/hadoop/GangliaMetrics +# +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +# Configuration of the "hbase" context for timeline metrics service +hbase.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +hbase.period=10 +hbase.collector={{timeline_server_hosts}}:8188 + +# Configuration of the "jvm" context for timeline metrics service +jvm.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +jvm.period=10 +jvm.collector={{timeline_server_hosts}}:8188 + +# Configuration of the "rpc" context for timeline metrics service +rpc.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +rpc.period=10 +rpc.collector={{timeline_server_hosts}}:8188 + +# Following hadoop example +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +hbase.sink.timeline.period=10 +hbase.sink.timeline.collector={{timeline_server_hosts}}:8188 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2.properties.j2 b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2.properties.j2 new file mode 100644 index 0000000..7a00a7e --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2.properties.j2 @@ -0,0 +1,58 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink|jmx].[instance].[options] +# See package.html for org.apache.hadoop.metrics2 for details + +{% if has_ganglia_server %} +*.period=60 + +*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink +*.sink.timeline.period=10 + + +# Hook up to the server +datanode.sink.timeline.collector={{timeline_server_hosts}}:8188 +namenode.sink.timeline.collector={{timeline_server_hosts}}:8188 +resourcemanager.sink.timeline.collector={{timeline_server_hosts}}:8188 +nodemanager.sink.timeline.collector={{timeline_server_hosts}}:8188 +historyserver.sink.timeline.collector={{timeline_server_hosts}}:8188 +journalnode.sink.timeline.collector={{timeline_server_hosts}}:8188 +nimbus.sink.timeline.collector={{timeline_server_hosts}}:8188 +supervisor.sink.timeline.collector={{timeline_server_hosts}}:8188 +maptask.sink.timeline.collector={{timeline_server_hosts}}:8188 +reducetask.sink.timeline.collector={{timeline_server_hosts}}:8188 + +resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue + +{% endif %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java new file mode 100644 index 0000000..2c42274 --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.sink.timeline; + +import org.apache.commons.configuration.SubsetConfiguration; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.metrics2.MetricsSink; +import org.apache.hadoop.metrics2.util.Servers; +import org.apache.hadoop.net.DNS; +import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.util.List; + +public abstract class AbstractTimelineMetricsSink implements MetricsSink { + + public final Log LOG = LogFactory.getLog(this.getClass()); + + private SubsetConfiguration conf; + private String hostName = "UNKNOWN.example.com"; + private String serviceName = ""; + private final String COLLECTOR_HOST_PROPERTY = "collector"; + private final int DEFAULT_PORT = 8188; + + private List<? extends SocketAddress> metricsServers; + private String collectorUri; + + @Override + public void init(SubsetConfiguration conf) { + this.conf = conf; + LOG.info("Initializing Timeline metrics sink."); + + // Take the hostname from the DNS class. + if (conf.getString("slave.host.name") != null) { + hostName = conf.getString("slave.host.name"); + } else { + try { + hostName = DNS.getDefaultHost( + conf.getString("dfs.datanode.dns.interface", "default"), + conf.getString("dfs.datanode.dns.nameserver", "default")); + } catch (UnknownHostException uhe) { + LOG.error(uhe); + hostName = "UNKNOWN.example.com"; + } + } + + serviceName = getFirstConfigPrefix(conf); + + // Load collector configs + metricsServers = Servers.parse(conf.getString(COLLECTOR_HOST_PROPERTY), + DEFAULT_PORT); + + if (metricsServers == null || metricsServers.isEmpty()) { + LOG.error("No Metric collector configured."); + } else { + collectorUri = "http://"; + conf.getString(COLLECTOR_HOST_PROPERTY).trim() + + "/ws/v1/timeline/metrics"; + } + } + + protected String getHostName() { + return hostName; + } + + protected String getServiceName() { + return serviceName; + } + + private String getFirstConfigPrefix(SubsetConfiguration conf) { + while (conf.getParent() instanceof SubsetConfiguration) { + conf = (SubsetConfiguration) conf.getParent(); + } + return conf.getPrefix(); + } + + protected SocketAddress getServerSocketAddress() { + if (metricsServers != null && !metricsServers.isEmpty()) { + return metricsServers.get(0); + } + return null; + } + + protected String getCollectorUri() { + return collectorUri; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java new file mode 100644 index 0000000..68b4be8 --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.sink.timeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Map; +import java.util.TreeMap; + +@XmlRootElement(name = "metric") +@XmlAccessorType(XmlAccessType.NONE) +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class TimelineMetric implements Comparable<TimelineMetric> { + + private String metricName; + private String appId; + private String instanceId; + private String hostName; + private long timestamp; + private long startTime; + private String type; + private Map<Long, Double> metricValues = new TreeMap<Long, Double>(); + + @XmlElement(name = "metricname") + public String getMetricName() { + return metricName; + } + + public void setMetricName(String metricName) { + this.metricName = metricName; + } + + @XmlElement(name = "appid") + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + @XmlElement(name = "instanceid") + public String getInstanceId() { + return instanceId; + } + + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + + @XmlElement(name = "hostname") + public String getHostName() { + return hostName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + @XmlElement(name = "timestamp") + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + @XmlElement(name = "starttime") + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + @XmlElement(name = "type") + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @XmlElement(name = "metrics") + public Map<Long, Double> getMetricValues() { + return metricValues; + } + + public void setMetricValues(Map<Long, Double> metricValues) { + this.metricValues = metricValues; + } + + public void addMetricValues(Map<Long, Double> metricValues) { + this.metricValues.putAll(metricValues); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TimelineMetric metric = (TimelineMetric) o; + + if (!metricName.equals(metric.metricName)) return false; + if (hostName != null ? !hostName.equals(metric.hostName) : metric.hostName != null) + return false; + if (appId != null ? !appId.equals(metric.appId) : metric.appId != null) + return false; + if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null) + return false; + if (timestamp != metric.timestamp) return false; + if (startTime != metric.startTime) return false; + + return true; + } + + public boolean equalsExceptTime(TimelineMetric metric) { + if (!metricName.equals(metric.metricName)) return false; + if (hostName != null ? !hostName.equals(metric.hostName) : metric.hostName != null) + return false; + if (appId != null ? !appId.equals(metric.appId) : metric.appId != null) + return false; + if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = metricName.hashCode(); + result = 31 * result + (appId != null ? appId.hashCode() : 0); + result = 31 * result + (instanceId != null ? instanceId.hashCode() : 0); + result = 31 * result + (hostName != null ? hostName.hashCode() : 0); + result = 31 * result + (int) (timestamp ^ (timestamp >>> 32)); + return result; + } + + @Override + public int compareTo(TimelineMetric other) { + if (timestamp > other.timestamp) { + return -1; + } else if (timestamp < other.timestamp) { + return 1; + } else { + return metricName.compareTo(other.metricName); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java new file mode 100644 index 0000000..a6c925a --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.sink.timeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +/** + * The class that hosts a list of timeline entities. + */ +@XmlRootElement(name = "metrics") +@XmlAccessorType(XmlAccessType.NONE) +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class TimelineMetrics { + + private List<TimelineMetric> allMetrics = new ArrayList<TimelineMetric>(); + + public TimelineMetrics() {} + + @XmlElement(name = "metrics") + public List<TimelineMetric> getMetrics() { + return allMetrics; + } + + public void setMetrics(List<TimelineMetric> allMetrics) { + this.allMetrics = allMetrics; + } + + private boolean isEqualTimelineMetrics(TimelineMetric metric1, + TimelineMetric metric2) { + + boolean isEqual = true; + + if (!metric1.getMetricName().equals(metric2.getMetricName())) { + return false; + } + + if (metric1.getHostName() != null) { + isEqual = metric1.getHostName().equals(metric2.getHostName()); + } + + if (metric1.getAppId() != null) { + isEqual = metric1.getAppId().equals(metric2.getAppId()); + } + + return isEqual; + } + + /** + * Merge with existing TimelineMetric if everything except startTime is + * the same. + * @param metric {@link TimelineMetric} + */ + public void addOrMergeTimelineMetric(TimelineMetric metric) { + TimelineMetric metricToMerge = null; + + if (!allMetrics.isEmpty()) { + for (TimelineMetric timelineMetric : allMetrics) { + if (timelineMetric.equalsExceptTime(metric)) { + metricToMerge = timelineMetric; + break; + } + } + } + + if (metricToMerge != null) { + metricToMerge.addMetricValues(metric.getMetricValues()); + if (metricToMerge.getTimestamp() > metric.getTimestamp()) { + metricToMerge.setTimestamp(metric.getTimestamp()); + } + if (metricToMerge.getStartTime() > metric.getStartTime()) { + metricToMerge.setStartTime(metric.getStartTime()); + } + } else { + allMetrics.add(metric); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricsCache.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricsCache.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricsCache.java new file mode 100644 index 0000000..36aaec2 --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricsCache.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.sink.timeline; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; + +import java.util.LinkedHashMap; +import java.util.Map; + +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class TimelineMetricsCache { + + private final TimelineMetricHolder timelineMetricCache = new TimelineMetricHolder(); + private static final Log LOG = LogFactory.getLog(TimelineMetric.class); + static final int MAX_RECS_PER_NAME_DEFAULT = 10000; + static final int MAX_EVICTION_TIME_MILLIS = 59000; // ~ 1 min + private final int maxRecsPerName; + private final int maxEvictionTimeInMillis; + + TimelineMetricsCache(int maxRecsPerName, int maxEvictionTimeInMillis) { + this.maxRecsPerName = maxRecsPerName; + this.maxEvictionTimeInMillis = maxEvictionTimeInMillis; + } + + class TimelineMetricWrapper { + private long timeDiff = -1; + private long oldestTimestamp = -1; + private TimelineMetric timelineMetric; + + TimelineMetricWrapper(TimelineMetric timelineMetric) { + this.timelineMetric = timelineMetric; + this.oldestTimestamp = timelineMetric.getStartTime(); + } + + private void updateTimeDiff(long timestamp) { + if (oldestTimestamp != -1 && timestamp > oldestTimestamp) { + timeDiff = timestamp - oldestTimestamp; + } else { + oldestTimestamp = timestamp; + } + } + + public void putMetric(TimelineMetric metric) { + this.timelineMetric.addMetricValues(metric.getMetricValues()); + updateTimeDiff(metric.getStartTime()); + } + + public long getTimeDiff() { + return timeDiff; + } + + public TimelineMetric getTimelineMetric() { + return timelineMetric; + } + } + + // TODO: Change to ConcurentHashMap with weighted eviction + class TimelineMetricHolder extends LinkedHashMap<String, TimelineMetricWrapper> { + private static final long serialVersionUID = 1L; + private boolean gotOverflow = false; + + @Override + protected boolean removeEldestEntry(Map.Entry<String, TimelineMetricWrapper> eldest) { + boolean overflow = size() > maxRecsPerName; + if (overflow && !gotOverflow) { + LOG.warn("Metrics cache overflow at "+ size() +" for "+ eldest); + gotOverflow = true; + } + return overflow; + } + + public TimelineMetric evict(String metricName) { + TimelineMetricWrapper metricWrapper = this.get(metricName); + + if (metricWrapper == null + || metricWrapper.getTimeDiff() < maxEvictionTimeInMillis) { + return null; + } + + TimelineMetric timelineMetric = metricWrapper.getTimelineMetric(); + this.remove(metricName); + + return timelineMetric; + } + + public void put(String metricName, TimelineMetric timelineMetric) { + + TimelineMetricWrapper metric = this.get(metricName); + if (metric == null) { + this.put(metricName, new TimelineMetricWrapper(timelineMetric)); + } else { + metric.putMetric(timelineMetric); + } + } + } + + public TimelineMetric getTimelineMetric(String metricName) { + if (timelineMetricCache.containsKey(metricName)) { + return timelineMetricCache.evict(metricName); + } + + return null; + } + + public void putTimelineMetric(TimelineMetric timelineMetric) { + timelineMetricCache.put(timelineMetric.getMetricName(), timelineMetric); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricsSink.java new file mode 100644 index 0000000..a843428 --- /dev/null +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricsSink.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.sink.timeline; + +import org.apache.commons.configuration.SubsetConfiguration; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.commons.httpclient.methods.StringRequestEntity; +import org.apache.commons.lang.ClassUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsException; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.impl.MsInfo; +import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsCache; +import org.codehaus.jackson.map.AnnotationIntrospector; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.codehaus.jackson.xc.JaxbAnnotationIntrospector; +import java.io.IOException; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class TimelineMetricsSink extends AbstractTimelineMetricsSink { + private static ObjectMapper mapper; + private Map<String, Set<String>> useTagsMap = new HashMap<String, Set<String>>(); + private static final String TAGS_FOR_PREFIX_PROPERTY_PREFIX = "tagsForPrefix."; + private static final String MAX_METRIC_ROW_CACHE_SIZE = "maxRowCacheSize"; + private static final String METRICS_SEND_INTERVAL = "sendInterval"; + protected HttpClient httpClient = new HttpClient(); + private TimelineMetricsCache metricsCache; + + static { + mapper = new ObjectMapper(); + AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); + mapper.setAnnotationIntrospector(introspector); + mapper.getSerializationConfig() + .setSerializationInclusion(JsonSerialize.Inclusion.NON_NULL); + } + + @Override + public void init(SubsetConfiguration conf) { + super.init(conf); + + int maxRowCacheSize = conf.getInt(MAX_METRIC_ROW_CACHE_SIZE, + TimelineMetricsCache.MAX_RECS_PER_NAME_DEFAULT); + int metricsSendInterval = conf.getInt(METRICS_SEND_INTERVAL, + TimelineMetricsCache.MAX_EVICTION_TIME_MILLIS); // ~ 1 min + metricsCache = new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval); + + conf.setListDelimiter(','); + Iterator<String> it = (Iterator<String>) conf.getKeys(); + while (it.hasNext()) { + String propertyName = it.next(); + if (propertyName != null && propertyName.startsWith(TAGS_FOR_PREFIX_PROPERTY_PREFIX)) { + String contextName = propertyName.substring(TAGS_FOR_PREFIX_PROPERTY_PREFIX.length()); + String[] tags = conf.getStringArray(propertyName); + boolean useAllTags = false; + Set<String> set = null; + if (tags.length > 0) { + set = new HashSet<String>(); + for (String tag : tags) { + tag = tag.trim(); + useAllTags |= tag.equals("*"); + if (tag.length() > 0) { + set.add(tag); + } + } + if (useAllTags) { + set = null; + } + } + useTagsMap.put(contextName, set); + } + } + } + + @Override + public void putMetrics(MetricsRecord record) { + try { + String recordName = record.name(); + String contextName = record.context(); + + StringBuilder sb = new StringBuilder(); + sb.append(contextName); + sb.append('.'); + sb.append(recordName); + + appendPrefix(record, sb); + sb.append("."); + int sbBaseLen = sb.length(); + + Collection<AbstractMetric> metrics = + (Collection<AbstractMetric>) record.metrics(); + + List<TimelineMetric> metricList = new ArrayList<TimelineMetric>(); + + for (AbstractMetric metric : metrics) { + sb.append(metric.name()); + String name = sb.toString(); + TimelineMetric timelineMetric = new TimelineMetric(); + timelineMetric.setMetricName(name); + timelineMetric.setHostName(getHostName()); + timelineMetric.setAppId(getServiceName()); + timelineMetric.setStartTime(record.timestamp()); + timelineMetric.setType(ClassUtils.getShortCanonicalName( + metric.value(), "Number")); + timelineMetric.getMetricValues().put(record.timestamp(), + metric.value().doubleValue()); + // Put intermediate values into the cache until it is time to send + metricsCache.putTimelineMetric(timelineMetric); + + // Retrieve all values from cache if it is time to send + TimelineMetric cachedMetric = metricsCache.getTimelineMetric(name); + + if (cachedMetric != null) { + metricList.add(cachedMetric); + } + + sb.setLength(sbBaseLen); + } + + TimelineMetrics timelineMetrics = new TimelineMetrics(); + timelineMetrics.setMetrics(metricList); + + if (!metricList.isEmpty()) { + emitMetrics(timelineMetrics); + } + + + } catch (IOException io) { + throw new MetricsException("Failed to putMetrics", io); + } + } + + private void emitMetrics(TimelineMetrics metrics) throws IOException { + String jsonData = mapper.writeValueAsString(metrics); + + SocketAddress socketAddress = getServerSocketAddress(); + + if (socketAddress != null) { + StringRequestEntity requestEntity = new StringRequestEntity( + jsonData, "application/json", "UTF-8"); + + PostMethod postMethod = new PostMethod(getCollectorUri()); + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode != 200) { + LOG.info("Unable to POST metrics to collector, " + getCollectorUri()); + } + } + } + + // Taken as is from Ganglia30 implementation + @InterfaceAudience.Private + public void appendPrefix(MetricsRecord record, StringBuilder sb) { + String contextName = record.context(); + Collection<MetricsTag> tags = record.tags(); + if (useTagsMap.containsKey(contextName)) { + Set<String> useTags = useTagsMap.get(contextName); + for (MetricsTag t : tags) { + if (useTags == null || useTags.contains(t.name())) { + + // the context is always skipped here because it is always added + + // the hostname is always skipped to avoid case-mismatches + // from different DNSes. + + if (t.info() != MsInfo.Context && t.info() != MsInfo.Hostname && t.value() != null) { + sb.append('.').append(t.name()).append('=').append(t.value()); + } + } + } + } + } + + @Override + public void flush() { + // TODO: Buffering implementation + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor b/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor new file mode 100644 index 0000000..fd086d9 --- /dev/null +++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor @@ -0,0 +1,203 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific + + +MONITOR_CONF_DIR=/etc/ambari-metrics-monitor/conf/ +METRIC_MONITOR=ambari-metrics-monitor + +RESOURCE_MONITORING_DIR=/usr/lib/python2.6/site-packages/resource_monitoring +METRIC_MONITOR_PY_SCRIPT=${RESOURCE_MONITORING_DIR}/main.py + +PIDFILE=/var/run/ambari-metrics-monitor/ambari-metrics-monitor.pid +OUTFILE=/var/log/ambari-metrics-monitor/ambari-metrics-monitor.out + +STOP_TIMEOUT=5 + +OK=0 +NOTOK=1 + +if [ -a /usr/bin/python2.7 ] && [ -z "${PYTHON}" ]; then + PYTHON=/usr/bin/python2.7 +fi + +if [ -a /usr/bin/python2.6 ] && [ -z "${PYTHON}" ]; then + PYTHON=/usr/bin/python2.6 +fi + +if [ "x$PYTHON" == "x" ]; then + PYTHON=/usr/bin/python +fi + +export PYTHON=${PYTHON} + +check_python_version () +{ + echo "Verifying Python version compatibility..." + majversion=`${PYTHON} -V 2>&1 | awk '{print $2}' | cut -d'.' -f1` + minversion=`${PYTHON} -V 2>&1 | awk '{print $2}' | cut -d'.' -f2` + numversion=$(( 10 * $majversion + $minversion)) + if (( $numversion < 26 )); then + echo "ERROR: Found Python version $majversion.$minversion. Ambari Metric Monitor requires Python version > 2.6" + return ${NOTOK} + fi + echo "Using python " ${PYTHON} + return ${OK} +} + +function write_pidfile +{ + local pidfile="$1" + echo $! > "${pidfile}" 2>/dev/null + if [[ $? -gt 0 ]]; then + echo "ERROR: Cannot write pid ${pidfile}." + exit 1; + fi +} + +#build psutil +if [ ! "$(ls -A ${RESOURCE_MONITORING_DIR}/psutil/build)" ]; then + echo "Building psutil..." + dir=$(pwd) + cd "${RESOURCE_MONITORING_DIR}/psutil" + ${PYTHON} "build.py" + cd "${dir}" +else + echo "psutil build directory is not empty, continuing..." +fi + +#locate config dir +while [[ -z "${_ams_configs_done}" ]]; do + case $1 in + --config) + shift + confdir=$1 + shift + if [[ -d "${confdir}" ]]; then + MONITOR_CONF_DIR="${confdir}" + elif [[ -z "${confdir}" ]]; then + echo "ERROR: No parameter provided for --config " + exit 1 + else + echo "ERROR: Cannot find configuration directory \"${confdir}\"" + exit 1 + fi + ;; + *) + _ams_configs_done=true + ;; + esac +done + +case "$1" in + + start) + check_python_version + if [ "$?" -eq "${NOTOK}" ]; then + exit -1 + fi + + echo "Checking for previously running Metric Monitor..." + if [ -f ${PIDFILE} ]; then + PID=`cat ${PIDFILE}` + if [ -z "`ps ax -o pid | grep ${PID}`" ]; then + echo "${PIDFILE} found with no process. Removing ${PID}..." + rm -f ${PIDFILE} + else + tput bold + echo "ERROR: ${METRIC_MONITOR} already running" + tput sgr0 + echo "Check ${PIDFILE} for PID." + exit -1 + fi + fi + + echo "Starting ${METRIC_MONITOR}" + + nohup ${PYTHON} ${METRIC_MONITOR_PY_SCRIPT} "$@" > ${OUTFILE} 2>&1 & + PID=$! + write_pidfile ${PIDFILE} + + sleep 2 + + echo "Verifying ${METRIC_MONITOR} process status..." + if [ -z "`ps ax -o pid | grep ${PID}`" ]; then + if [ -s ${OUTFILE} ]; then + echo "ERROR: ${METRIC_MONITOR} start failed. For more details, see ${OUTFILE}:" + echo "====================" + tail -n 10 ${OUTFILE} + echo "====================" + else + echo "ERROR: ${METRIC_MONITOR} start failed" + rm -f ${PIDFILE} + fi + echo "Monitor out at: ${OUTFILE}" + exit -1 + fi + + echo "Metric Monitor successfully started" + echo "Server log at: ${OUTFILE}" + ;; + status) + if [ -f ${PIDFILE} ]; then + PID=`cat ${PIDFILE}` + echo "Found ${METRIC_MONITOR} PID: $PID" + if [ -z "`ps ax -o pid | grep ${PID}`" ]; then + echo "${METRIC_MONITOR} not running. Stale PID File at: $PIDFILE" + retcode=2 + else + tput bold + echo "${METRIC_MONITOR} running." + tput sgr0 + echo "Monitor PID at: ${PIDFILE}" + echo "Monitor out at: ${OUTFILE}" + fi + else + tput bold + echo "${METRIC_MONITOR} currently not running" + tput sgr0 + echo "Usage: /usr/sbin/${METRIC_MONITOR} {start|stop|restart|status}" + retcode=3 + fi + ;; + stop) + pidfile=${PIDFILE} + + if [[ -f "${pidfile}" ]]; then + pid=$(cat "$pidfile") + + kill "${pid}" >/dev/null 2>&1 + sleep "${STOP_TIMEOUT}" + + if kill -0 "${pid}" > /dev/null 2>&1; then + echo "WARNING: ${METRIC_MONITOR} did not stop gracefully after ${STOP_TIMEOUT} seconds: Trying to kill with kill -9" + kill -9 "${pid}" >/dev/null 2>&1 + fi + + if ps -p "${pid}" > /dev/null 2>&1; then + echo "ERROR: Unable to kill ${pid}" + else + rm -f "${pidfile}" >/dev/null 2>&1 + fi + fi + + ;; + restart) + echo -e "Restarting ${METRIC_MONITOR}" + $0 stop + $0 start "$@" + retcode=$? + ;; +esac http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-host-monitoring/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/pom.xml b/ambari-metrics/ambari-metrics-host-monitoring/pom.xml new file mode 100644 index 0000000..b354461 --- /dev/null +++ b/ambari-metrics/ambari-metrics-host-monitoring/pom.xml @@ -0,0 +1,273 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <parent> + <artifactId>ambari-metrics</artifactId> + <groupId>org.apache.ambari</groupId> + <version>0.1.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <packaging>pom</packaging> + <version>0.1.0-SNAPSHOT</version> + <artifactId>ambari-metrics-host-monitoring</artifactId> + <properties> + <resmonitor.install.dir> + /usr/lib/python2.6/site-packages/resource_monitoring + </resmonitor.install.dir> + </properties> + <build> + <plugins> + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <version>2.6</version> + <configuration> + <filesets> + <fileset> + <directory>${project.basedir}/src/main/python/psutil/build/</directory> + <includes> + <include>**/*</include> + </includes> + <followSymlinks>false</followSymlinks> + </fileset> + </filesets> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.8</version> + <executions> + <execution> + <id>parse-version</id> + <phase>validate</phase> + <goals> + <goal>parse-version</goal> + </goals> + </execution> + <execution> + <id>regex-property</id> + <goals> + <goal>regex-property</goal> + </goals> + <configuration> + <name>ambariVersion</name> + <value>${project.version}</value> + <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex> + <replacement>$1.$2.$3</replacement> + <failIfNoMatch>false</failIfNoMatch> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.0</version> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <tarLongFileMode>gnu</tarLongFileMode> + <descriptors> + <descriptor>${project.basedir}/../../ambari-project/src/main/assemblies/empty.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>build-tarball</id> + <phase>none</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>rpm-maven-plugin</artifactId> + <version>2.0.1</version> + <executions> + <execution> + <!-- unbinds rpm creation from maven lifecycle --> + <phase>none</phase> + <goals> + <goal>rpm</goal> + </goals> + </execution> + </executions> + + <configuration> + <name>ambari-metrics-monitor</name> + <group>Development</group> + <needarch>x86_64</needarch> + <autoRequires>false</autoRequires> + <requires> + <require>ambari-metrics-hadoop-sink</require> + <require>${python.ver}</require> + <require>gcc</require> + <require>python-devel</require> + </requires> + <preremoveScriptlet> + <scriptFile>src/main/package/rpm/preremove.sh</scriptFile> + <fileEncoding>utf-8</fileEncoding> + </preremoveScriptlet> + <mappings> + <mapping> + <directory>${resmonitor.install.dir}</directory> + <username>root</username> + <groupname>root</groupname> + <sources> + <source> + <!--<location>--> + <!--${project.build.directory}/${project.artifactId}-${project.version}/resource_monitoring/external/build/*--> + <!--</location>--> + <!--<location>--> + <!--${project.build.directory}/${project.artifactId}-${project.version}/resource_monitoring/core--> + <!--</location>--> + <!--<location>--> + <!--${project.build.directory}/${project.artifactId}-${project.version}/resource_monitoring/__init__.py--> + <!--</location>--> + <location> + ${project.basedir}/src/main/python/__init__.py + </location> + </source> + <source> + <location> + ${project.basedir}/src/main/python/main.py + </location> + </source> + </sources> + </mapping> + <mapping> + <directory>${resmonitor.install.dir}/core</directory> + <sources> + <source> + <location> + ${project.basedir}/src/main/python/core + </location> + </source> + </sources> + </mapping> + <mapping> + <directory>${resmonitor.install.dir}/psutil</directory> + <sources> + <source> + <location> + ${project.basedir}/src/main/python/psutil + </location> + </source> + </sources> + </mapping> + <mapping> + <directory>/etc/ambari-metrics-monitor/conf</directory> + <configuration>true</configuration> + </mapping> + <mapping> + <directory>/var/run/ambari-metrics-monitor</directory> + </mapping> + <mapping> + <directory>/var/log/ambari-metrics-monitor</directory> + </mapping> + <mapping> + <directory>/usr/sbin</directory> + <filemode>755</filemode> + <username>root</username> + <groupname>root</groupname> + <directoryIncluded>false</directoryIncluded> + <sources> + <source> + <location>conf/unix/ambari-metrics-monitor</location> + <filter>true</filter> + </source> + </sources> + </mapping> + </mappings> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <id>psutils-compile</id> + <phase>process-classes</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target name="psutils-compile"> + <exec dir="${basedir}/src/main/python/psutil" executable="python" failonerror="true"> + <arg value="setup.py" /> + <arg value="build" /> + <arg value="--build-platlib" /> + <arg value="${basedir}/target/psutil_build" /> + </exec> + </target> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <configuration> + <executable>python</executable> + <workingDirectory>src/test/python</workingDirectory> + <arguments> + <argument>unitTests.py</argument> + </arguments> + <environmentVariables> + <PYTHONPATH>../../main/python:$PYTHONPATH</PYTHONPATH> + </environmentVariables> + <skip>${skipTests}</skip> + </configuration> + <id>python-test</id> + <phase>test</phase> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>src/main/python/psutil/**</exclude> + <exclude>.pydevproject</exclude> + </excludes> + </configuration> + <executions> + <execution> + <phase>test</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-host-monitoring/src/main/package/rpm/preremove.sh ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/package/rpm/preremove.sh b/ambari-metrics/ambari-metrics-host-monitoring/src/main/package/rpm/preremove.sh new file mode 100644 index 0000000..9789127 --- /dev/null +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/package/rpm/preremove.sh @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# WARNING: This script is performed not only on uninstall, but also +# during package update. See http://www.ibm.com/developerworks/library/l-rpm2/ +# for details + +RESOURCE_MONITORING_DIR=/usr/lib/python2.6/site-packages/resource_monitoring +PSUTIL_DIR="${RESOURCE_MONITORING_DIR}/psutil" + + +if [ -d "${PSUTIL_DIR}" ]; then + rm -rf "${PSUTIL_DIR}/*" +fi + +exit 0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/__init__.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/__init__.py new file mode 100644 index 0000000..adfc71c --- /dev/null +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from core import * http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py new file mode 100644 index 0000000..996120f --- /dev/null +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os, sys +path = os.path.abspath(__file__) +path = os.path.join(os.path.dirname(os.path.dirname(path)), "psutil/build/") + +for dir in os.walk(path).next()[1]: + if 'lib' in dir: + sys.path.append(os.path.join(path, dir)) + +try: + import psutil +except ImportError: + print 'psutil binaries need to be built by running, psutil/build.py ' \ + 'manually or by running a, mvn clean package, command.'
