Windows build for 2 Ambari Metrics service: Host Monitoring and Timeline Service (Collector).
+ Added Windows profiles to the Maven project files + Added the necessary Windows assemblies + Created Windows service skeletons + Host Monitoring: added OS-independent process termination handler + Collector: added debugging support for the Java process + Fixed services shutdown, especially when joining spawned threads + Fixed unit tests + Added support for unit testing on MacOS and Windows Windows-specific: + Moved the assembly descriptors to ambari-metrics-assembly + Fixed comments in the configuration files + Added soft dependencies on the embedded HBase service + Added support for the embedded HBase service setup Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6c21b094 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6c21b094 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6c21b094 Branch: refs/heads/trunk Commit: 6c21b0942f40791ab4a461048d55e975593eab85 Parents: 9884cbd Author: Florian Barca <fba...@hortonworks.com> Authored: Tue Dec 30 10:35:45 2014 -0800 Committer: Florian Barca <fba...@hortonworks.com> Committed: Tue Dec 30 10:35:45 2014 -0800 ---------------------------------------------------------------------- ambari-agent/conf/windows/service_wrapper.py | 2 +- .../src/main/python/ambari_commons/os_utils.py | 17 +- .../main/python/ambari_commons/os_windows.py | 21 +- .../src/main/python/ambari_commons/xml_utils.py | 33 +++ ambari-metrics/ambari-metrics-assembly/pom.xml | 77 ++++++- .../src/main/assembly/collector-windows.xml | 101 ++++++++ .../src/main/assembly/monitor-windows.xml | 84 +++++++ .../src/main/assembly/monitor.xml | 2 +- .../src/main/assembly/sink-windows.xml | 60 +++++ .../conf/windows/ambari-metrics-monitor.cmd | 17 ++ .../conf/windows/metric_groups.conf | 19 ++ .../conf/windows/metric_monitor.ini | 30 +++ .../ambari-metrics-host-monitoring/pom.xml | 146 ++++++++---- .../src/main/python/amhm_service.py | 231 +++++++++++++++++++ .../src/main/python/core/__init__.py | 3 +- .../src/main/python/core/config_reader.py | 66 +++++- .../src/main/python/core/controller.py | 27 ++- .../src/main/python/core/emitter.py | 17 +- .../src/main/python/core/stop_handler.py | 138 +++++++++++ .../src/main/python/main.py | 58 ++++- .../src/test/python/core/TestEmitter.py | 41 ++-- .../conf/windows/ambari-metrics-collector.cmd | 17 ++ .../conf/windows/ams-env.cmd | 16 ++ .../conf/windows/ams-site.xml | 25 ++ .../conf/windows/ams.properties | 17 ++ .../conf/windows/log4j.properties | 29 +++ .../ambari-metrics-timelineservice/pom.xml | 31 +++ .../python/ambari_metrics_collector/__init__.py | 21 ++ .../ambari_metrics_collector/properties.py | 223 ++++++++++++++++++ .../serviceConfiguration.py | 152 ++++++++++++ .../src/main/python/amc_service.py | 174 ++++++++++++++ .../src/main/python/embedded_hbase_service.py | 201 ++++++++++++++++ .../src/main/python/main.py | 214 +++++++++++++++++ ambari-metrics/pom.xml | 27 +++ .../main/python/ambari-server-state/Entities.py | 17 +- pom.xml | 2 + 36 files changed, 2238 insertions(+), 118 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-agent/conf/windows/service_wrapper.py ---------------------------------------------------------------------- diff --git a/ambari-agent/conf/windows/service_wrapper.py b/ambari-agent/conf/windows/service_wrapper.py index 40be1d0..5eb06c4 100644 --- a/ambari-agent/conf/windows/service_wrapper.py +++ b/ambari-agent/conf/windows/service_wrapper.py @@ -92,7 +92,7 @@ class AmbariAgentService(AmbariService): # Soft dependency on the Windows Time service ensure_time_service_is_started() - self.heartbeat_stop_handler = HeartbeatStopHandlers(self._heventSvcStop) + self.heartbeat_stop_handler = HeartbeatStopHandlers(AmbariAgentService._heventSvcStop) self.ReportServiceStatus(win32service.SERVICE_RUNNING) http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-common/src/main/python/ambari_commons/os_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_utils.py b/ambari-common/src/main/python/ambari_commons/os_utils.py index 3f4819d..942a920 100644 --- a/ambari-common/src/main/python/ambari_commons/os_utils.py +++ b/ambari-common/src/main/python/ambari_commons/os_utils.py @@ -48,14 +48,17 @@ def quote_path(filepath): filepath_ret = filepath return filepath_ret -def search_file(filename, search_path, pathsep=os.pathsep): - """ Given a search path, find file with requested name """ +def _search_file(filename, search_path, pathsep): for path in string.split(search_path, pathsep): candidate = os.path.join(path, filename) if os.path.exists(candidate): return os.path.abspath(candidate) return None +def search_file(filename, search_path, pathsep=os.pathsep): + """ Given a search path, find file with requested name """ + return _search_file(filename, search_path, pathsep) + def copy_file(src, dest_file): try: shutil.copyfile(src, dest_file) @@ -103,9 +106,7 @@ def get_password(prompt): return os_getpass(prompt) def find_in_path(file): - dirs = os.environ["PATH"].split(os.pathsep) - for dir in dirs: - full_path = os.path.join(dir, file) - if os.path.exists(full_path): - return full_path - raise Exception("File {} not found in PATH".format(file)) \ No newline at end of file + full_path = _search_file(file, os.environ["PATH"], os.pathsep) + if full_path is None: + raise Exception("File {0} not found in PATH".format(file)) + return full_path \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-common/src/main/python/ambari_commons/os_windows.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/os_windows.py b/ambari-common/src/main/python/ambari_commons/os_windows.py index 2fb98e4..7918a2f 100644 --- a/ambari-common/src/main/python/ambari_commons/os_windows.py +++ b/ambari-common/src/main/python/ambari_commons/os_windows.py @@ -402,7 +402,7 @@ class WinService(win32serviceutil.ServiceFramework): # _svc_display_name_ = The service display name # _svc_description_ = The service description - _heventSvcStop = win32event.CreateEvent(None, 0, 0, None) + _heventSvcStop = win32event.CreateEvent(None, 1, 0, None) _hmtxOut = win32event.CreateMutex(None, False, None) #[fbarca] Python doesn't support critical sections def __init__(self, *args): @@ -418,17 +418,18 @@ class WinService(win32serviceutil.ServiceFramework): def SvcStop(self): self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) - win32event.SetEvent(self._heventSvcStop) + win32event.SetEvent(WinService._heventSvcStop) # Service code entry point. Override it to implement the intended functionality. def ServiceMain(self): #Default implementation, does nothing. - win32event.WaitForSingleObject(self._heventSvcStop, win32event.INFINITE) + win32event.WaitForSingleObject(WinService._heventSvcStop, win32event.INFINITE) pass - def DefCtrlCHandler(self): + @staticmethod + def DefCtrlCHandler(): print_info_msg("Ctrl+C handler invoked. Stopping.") - win32event.SetEvent(self._heventSvcStop) + win32event.SetEvent(WinService._heventSvcStop) pass #username domain\\username : The Username the service is to run under @@ -438,7 +439,7 @@ class WinService(win32serviceutil.ServiceFramework): #perfmonini file: .ini file to use for registering performance monitor data #perfmondll file: .dll file to use when querying the service for performance data, default = perfmondata.dll @classmethod - def Install(cls, startupMode = "auto", username = None, password = None, interactive = False, + def Install(cls, classPath = None, startupMode = "auto", username = None, password = None, interactive = False, perfMonIni = None, perfMonDll = None): installArgs = [sys.argv[0], "--startup=" + startupMode] if username is not None and username: @@ -452,7 +453,8 @@ class WinService(win32serviceutil.ServiceFramework): if perfMonDll is not None and perfMonDll: installArgs.append("--perfmondll=" + perfMonDll) installArgs.append("install") - win32serviceutil.HandleCommandLine(cls, None, installArgs) + + win32serviceutil.HandleCommandLine(cls, classPath, installArgs) @classmethod def Start(cls, waitSecs = 30): @@ -483,11 +485,12 @@ class WinService(win32serviceutil.ServiceFramework): def CheckForStop(self): #Check for stop event to be signaled - return win32event.WAIT_OBJECT_0 == win32event.WaitForSingleObject(self._heventSvcStop, 1) + return win32event.WAIT_OBJECT_0 == win32event.WaitForSingleObject(WinService._heventSvcStop, 1) def _StopOrWaitForChildProcessToFinish(self, childProcess): #Wait for the child process to finish or for the stop event to be signaled - if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([self._heventSvcStop, childProcess._handle], False, win32event.INFINITE)): + if(win32event.WAIT_OBJECT_0 == win32event.WaitForMultipleObjects([WinService._heventSvcStop, childProcess._handle], + False, win32event.INFINITE)): # The OS only detaches the child process when the master process exits. # We must kill it manually. try: http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-common/src/main/python/ambari_commons/xml_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/xml_utils.py b/ambari-common/src/main/python/ambari_commons/xml_utils.py new file mode 100644 index 0000000..31b2968 --- /dev/null +++ b/ambari-common/src/main/python/ambari_commons/xml_utils.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#* Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import os + +class ConvertToXml: + "Template class, allow to output fields in xml format" + def getField(self): + return [name for name, obj in inspect.getmembers(self) + if not name.startswith("__") and not inspect.isroutine(obj)] + + def attributesToXml(self): + result = "" + listOfAttr = self.getField() + for attrName in listOfAttr: + result += "<" + attrName + ">" + result += str(getattr(self, attrName)) + result += "</" + attrName + ">\n" + return result http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml index 1e020cb..0a47b8a 100644 --- a/ambari-metrics/ambari-metrics-assembly/pom.xml +++ b/ambari-metrics/ambari-metrics-assembly/pom.xml @@ -26,9 +26,9 @@ <version>0.1.0-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>ambari-metrics-assembly</artifactId> <packaging>pom</packaging> + <version>0.1.0-SNAPSHOT</version> <properties> <collector.dir>${project.basedir}/../ambari-metrics-timelineservice</collector.dir> @@ -48,6 +48,33 @@ <build> <plugins> <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.8</version> + <executions> + <execution> + <id>parse-version</id> + <phase>validate</phase> + <goals> + <goal>parse-version</goal> + </goals> + </execution> + <execution> + <id>regex-property</id> + <goals> + <goal>regex-property</goal> + </goals> + <configuration> + <name>ambariVersion</name> + <value>${project.version}</value> + <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex> + <replacement>$1.$2.$3</replacement> + <failIfNoMatch>false</failIfNoMatch> + </configuration> + </execution> + </executions> + </plugin> + <plugin> <artifactId>maven-resources-plugin</artifactId> <version>2.7</version> @@ -94,7 +121,7 @@ <finalName>ambari-metrics-collector-${project.version}</finalName> <appendAssemblyId>false</appendAssemblyId> <descriptors> - <descriptor>src/main/assembly/collector.xml</descriptor> + <descriptor>${assemblydescriptor.collector}</descriptor> </descriptors> <tarLongFileMode>gnu</tarLongFileMode> </configuration> @@ -110,7 +137,7 @@ <finalName>ambari-metrics-monitor-${project.version}</finalName> <appendAssemblyId>false</appendAssemblyId> <descriptors> - <descriptor>src/main/assembly/monitor.xml</descriptor> + <descriptor>${assemblydescriptor.monitor}</descriptor> </descriptors> <tarLongFileMode>gnu</tarLongFileMode> </configuration> @@ -126,7 +153,7 @@ <finalName>ambari-metrics-hadoop-sink-${project.version}</finalName> <appendAssemblyId>false</appendAssemblyId> <descriptors> - <descriptor>src/main/assembly/sink.xml</descriptor> + <descriptor>${assemblydescriptor.sink}</descriptor> </descriptors> <tarLongFileMode>gnu</tarLongFileMode> </configuration> @@ -710,6 +737,48 @@ </plugins> </build> </profile> + <profile> + <id>windows</id> + <activation> + <os> + <family>win</family> + </os> + </activation> + <properties> + <envClassifier>win</envClassifier> + <dirsep>\</dirsep> + <pathsep>;</pathsep> + <executable.python>python</executable.python> + <executable.shell>cmd</executable.shell> + <fileextension.shell>cmd</fileextension.shell> + <fileextension.dot.shell-default>.cmd</fileextension.dot.shell-default> + <assemblydescriptor.collector>src/main/assembly/collector-windows.xml</assemblydescriptor.collector> + <assemblydescriptor.monitor>src/main/assembly/monitor-windows.xml</assemblydescriptor.monitor> + <assemblydescriptor.sink>src/main/assembly/sink-windows.xml</assemblydescriptor.sink> + <packagingFormat>jar</packagingFormat> + </properties> + </profile> + <profile> + <id>linux</id> + <activation> + <os> + <family>unix</family> + </os> + </activation> + <properties> + <envClassifier>linux</envClassifier> + <dirsep>/</dirsep> + <pathsep>:</pathsep> + <executable.python>${project.basedir}/../ambari-common/src/main/unix/ambari-python-wrap</executable.python> + <executable.shell>sh</executable.shell> + <fileextension.shell>sh</fileextension.shell> + <fileextension.dot.shell-default></fileextension.dot.shell-default> + <assemblydescriptor.collector>src/main/assembly/collector.xml</assemblydescriptor.collector> + <assemblydescriptor.monitor>src/main/assembly/monitor.xml</assemblydescriptor.monitor> + <assemblydescriptor.sink>src/main/assembly/sink.xml</assemblydescriptor.sink> + <packagingFormat>jar</packagingFormat> + </properties> + </profile> </profiles> http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml new file mode 100644 index 0000000..3faa085 --- /dev/null +++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml @@ -0,0 +1,101 @@ +<?xml version="1.0"?> + +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd";> + <id>collector</id> + <formats> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${collector.dir}/target/embedded/${hbase.folder}</directory> + <outputDirectory>hbase</outputDirectory> + </fileSet> + <fileSet> + <directory>${collector.dir}/conf/windows</directory> + <outputDirectory>/</outputDirectory> + <includes> + <include>ambari-metrics-collector.cmd</include> + </includes> + </fileSet> + <fileSet> + <directory>${collector.dir}/conf/windows</directory> + <outputDirectory>conf</outputDirectory> + <includes> + <include>ams.properties</include> + <include>ams-env.cmd</include> + <include>ams-site.xml</include> + <include>log4j.properties</include> + </includes> + </fileSet> + <fileSet> + <directory>${collector.dir}/target/lib</directory> + <outputDirectory>lib</outputDirectory> + </fileSet> + <fileSet> + <directory>${collector.dir}/src/main/python</directory> + <outputDirectory>/sbin</outputDirectory> + <includes> + <include>*.py</include> + </includes> + </fileSet> + <fileSet> + <directory>${collector.dir}/src/main/python/ambari_metrics_collector</directory> + <outputDirectory>/sbin/ambari_metrics_collector</outputDirectory> + <includes> + <include>*.py</include> + </includes> + </fileSet> + <fileSet> + <directory>${project.basedir}/../../ambari-common/src/main/python/ambari_commons</directory> + <outputDirectory>/sbin/ambari_commons</outputDirectory> + <includes> + <include>*.py</include> + </includes> + </fileSet> + <fileSet> + <directory>${project.basedir}/../../ambari-common/src/main/python/ambari_commons/resources</directory> + <outputDirectory>/sbin/ambari_commons/resources</outputDirectory> + <includes> + <include>*.json</include> + </includes> + </fileSet> + </fileSets> + <dependencySets> + <dependencySet> + <unpack>false</unpack> + <outputDirectory>hbase/lib</outputDirectory> + <useProjectArtifact>false</useProjectArtifact> + <includes> + <include>org.antlr:antlr*</include> + <include>org.apache.phoenix:phoenix-core</include> + </includes> + </dependencySet> + <dependencySet> + <unpack>false</unpack> + <outputDirectory>lib</outputDirectory> + <useProjectArtifact>false</useProjectArtifact> + <includes> + <include>org.apache.ambari:ambari-metrics-timelineservice</include> + </includes> + </dependencySet> + </dependencySets> +</assembly> http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml new file mode 100644 index 0000000..65a936b --- /dev/null +++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor-windows.xml @@ -0,0 +1,84 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>windows-dist</id> + <formats> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${monitor.dir}/src/main/python</directory> + <outputDirectory>/sbin</outputDirectory> + <includes> + <include>*.py</include> + </includes> + </fileSet> + <fileSet> + <directory>${project.basedir}/../../ambari-common/src/main/python/ambari_commons</directory> + <outputDirectory>/sbin/ambari_commons</outputDirectory> + <includes> + <include>*.py</include> + </includes> + </fileSet> + <fileSet> + <directory>${project.basedir}/../../ambari-common/src/main/python/ambari_commons/resources</directory> + <outputDirectory>/sbin/ambari_commons/resources</outputDirectory> + <includes> + <include>*.json</include> + </includes> + </fileSet> + <fileSet> + <directory>${monitor.dir}/src/main/python/core</directory> + <outputDirectory>/sbin/core</outputDirectory> + <includes> + <include>*.py</include> + </includes> + </fileSet> + <fileSet> + <directory>${monitor.dir}/target/psutil_build</directory> + <outputDirectory>/sbin/core/psutil/build</outputDirectory> + <includes> + <include>*.egg</include> + </includes> + </fileSet> + <fileSet> + <directory>${monitor.dir}/conf/windows</directory> + <outputDirectory>conf</outputDirectory> + <includes> + <include>metric_groups.conf</include> + <include>metric_monitor.ini</include> + </includes> + </fileSet> + <fileSet> + <directory>${monitor.dir}/conf/windows</directory> + <outputDirectory>/</outputDirectory> + <includes> + <include>ambari-metrics-monitor.cmd</include> + </includes> + </fileSet> + </fileSets> + <dependencySets> + <dependencySet> + <useProjectArtifact>false</useProjectArtifact> + <excludes> + <exclude>*</exclude> + </excludes> + </dependencySet> + </dependencySets> +</assembly> http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml index 3513972..5944616 100644 --- a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml +++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/monitor.xml @@ -31,8 +31,8 @@ <outputDirectory>site-packages/resource_monitoring</outputDirectory> </fileSet> <fileSet> - <outputDirectory>conf</outputDirectory> <directory>${monitor.dir}/conf/unix</directory> + <outputDirectory>conf</outputDirectory> <includes> <include>metric_groups.conf</include> <include>metric_monitor.ini</include> http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-assembly/src/main/assembly/sink-windows.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/sink-windows.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/sink-windows.xml new file mode 100644 index 0000000..a65d6f2 --- /dev/null +++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/sink-windows.xml @@ -0,0 +1,60 @@ +<?xml version="1.0"?> + +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd";> + <id>hadoop-sink</id> + <formats> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${hadoop-sink.dir}/src/main/conf</directory> + <outputDirectory>hadoop-sink/conf</outputDirectory> + </fileSet> + <fileSet> + <directory>${flume-sink.dir}/src/main/conf</directory> + <outputDirectory>hadoop-sink/conf</outputDirectory> + </fileSet> + <fileSet> + <directory>${storm-sink.dir}/src/main/conf</directory> + <outputDirectory>hadoop-sink/conf</outputDirectory> + </fileSet> + </fileSets> + + <files> + <file> + <source>${hadoop-sink.dir}/target/ambari-metrics-hadoop-sink-with-common-${project.version}.jar</source> + <outputDirectory>hadoop-sink</outputDirectory> + </file> + <file> + <source>${flume-sink.dir}/target/ambari-metrics-flume-sink-with-common-${project.version}.jar</source> + <outputDirectory>hadoop-sink</outputDirectory> + </file> + <file> + <source>${storm-sink.dir}/target/ambari-metrics-storm-sink-with-common-${project.version}.jar</source> + <outputDirectory>hadoop-sink</outputDirectory> + </file> + </files> + + + + +</assembly> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/ambari-metrics-monitor.cmd ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/ambari-metrics-monitor.cmd b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/ambari-metrics-monitor.cmd new file mode 100644 index 0000000..115b6a6 --- /dev/null +++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/ambari-metrics-monitor.cmd @@ -0,0 +1,17 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@echo off +python.exe -u %~dp0\sbin\amhm_service.py %* 2>&1 http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_groups.conf ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_groups.conf b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_groups.conf new file mode 100644 index 0000000..6d5a62f --- /dev/null +++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_groups.conf @@ -0,0 +1,19 @@ +{ + "host_metric_groups": { + "all": { + "collect_every": "10", + "metrics": [ + { + "name": "bytes_out", + "value_threshold": "128" + } + ] + } + }, + "process_metric_groups": { + "": { + "collect_every": "15", + "metrics": [] + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_monitor.ini ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_monitor.ini b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_monitor.ini new file mode 100644 index 0000000..bc2b461 --- /dev/null +++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/windows/metric_monitor.ini @@ -0,0 +1,30 @@ +; +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; + +[default] +debug_level = INFO +metrics_server = {{ams_collector_host_single}}:{{ams_collector_port}} +enable_time_threshold = false +enable_value_threshold = false + +[emitter] +send_interval = 60 + +[collector] +collector_sleep_interval = 5 +max_queue_size = 5000 http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/pom.xml b/ambari-metrics/ambari-metrics-host-monitoring/pom.xml index c213057..c2f322c 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/pom.xml +++ b/ambari-metrics/ambari-metrics-host-monitoring/pom.xml @@ -32,6 +32,7 @@ <resmonitor.install.dir> /usr/lib/python2.6/site-packages/resource_monitoring </resmonitor.install.dir> + <final.name>${project.artifactId}-${project.version}</final.name> </properties> <build> <plugins> @@ -82,55 +83,12 @@ <version>3.0</version> </plugin> <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <tarLongFileMode>gnu</tarLongFileMode> - <descriptors> - <descriptor>${project.basedir}/../../ambari-project/src/main/assemblies/empty.xml</descriptor> - </descriptors> - </configuration> - <executions> - <execution> - <id>build-tarball</id> - <phase>none</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <version>1.7</version> - <executions> - <execution> - <id>psutils-compile</id> - <phase>process-test-classes</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <target name="psutils-compile"> - <exec dir="${basedir}/src/main/python/psutil" executable="${project.basedir}/../../ambari-common/src/main/unix/ambari-python-wrap" failonerror="true"> - <arg value="setup.py" /> - <arg value="build" /> - <arg value="--build-platlib" /> - <arg value="${basedir}/target/psutil_build" /> - </exec> - </target> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>exec-maven-plugin</artifactId> <executions> <execution> <configuration> - <executable>${project.basedir}/../../ambari-common/src/main/unix/ambari-python-wrap</executable> + <executable>${executable.python}</executable> <workingDirectory>src/test/python</workingDirectory> <arguments> <argument>unitTests.py</argument> @@ -153,6 +111,8 @@ <artifactId>apache-rat-plugin</artifactId> <configuration> <excludes> + <exclude>conf/unix/metric_groups.conf</exclude> + <exclude>conf/windows/metric_groups.conf</exclude> <exclude>src/main/python/psutil/**</exclude> <exclude>.pydevproject</exclude> </excludes> @@ -168,4 +128,102 @@ </plugin> </plugins> </build> + <profiles> + <profile> + <id>windows</id> + <activation> + <os> + <family>win</family> + </os> + </activation> + <properties> + <envClassifier>win</envClassifier> + <dirsep>\</dirsep> + <pathsep>;</pathsep> + <executable.python>python</executable.python> + <executable.shell>cmd</executable.shell> + <fileextension.shell>cmd</fileextension.shell> + <fileextension.dot.shell-default>.cmd</fileextension.dot.shell-default> + <assemblydescriptor>src/main/assemblies/amhm-windows.xml</assemblydescriptor> + <packagingFormat>jar</packagingFormat> + </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <id>psutils-compile</id> + <phase>process-test-classes</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target name="psutils-compile"> + <exec dir="${basedir}/src/main/python/psutil" executable="python" failonerror="true"> + <arg value="setup.py" /> + <arg value="bdist_egg" /> + <arg value="--bdist-dir" /> + <arg value="${basedir}/target/psutil_build_temp" /> + <arg value="--dist-dir" /> + <arg value="${basedir}/target/psutil_build" /> + </exec> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>linux</id> + <activation> + <os> + <family>unix</family> + </os> + </activation> + <properties> + <envClassifier>linux</envClassifier> + <dirsep>/</dirsep> + <pathsep>:</pathsep> + <executable.python>${project.basedir}/../../ambari-common/src/main/unix/ambari-python-wrap</executable.python> + <executable.shell>sh</executable.shell> + <fileextension.shell>sh</fileextension.shell> + <fileextension.dot.shell-default></fileextension.dot.shell-default> + <assemblydescriptor>src/main/assemblies/empty.xml</assemblydescriptor> + <packagingFormat>jar</packagingFormat> + </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <id>psutils-compile</id> + <phase>process-test-classes</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target name="psutils-compile"> + <exec dir="${basedir}/src/main/python/psutil" executable="python" failonerror="true"> + <arg value="setup.py" /> + <arg value="build" /> + <arg value="--build-platlib" /> + <arg value="${basedir}/target/psutil_build" /> + </exec> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/amhm_service.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/amhm_service.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/amhm_service.py new file mode 100644 index 0000000..0f8daab --- /dev/null +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/amhm_service.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import glob + +import optparse +import os +import sys + +from ambari_commons.ambari_service import AmbariService +from ambari_commons.exceptions import FatalException, NonFatalException +from ambari_commons.logging_utils import print_warning_msg, print_error_msg, print_info_msg +from ambari_commons.os_utils import search_file, run_os_command +from ambari_commons.os_windows import SvcStatusCallback +from core.config_reader import SERVER_OUT_FILE, SERVICE_USERNAME_KEY, SERVICE_PASSWORD_KEY, \ + SETUP_ACTION, START_ACTION, STOP_ACTION, RESTART_ACTION, STATUS_ACTION +from core.stop_handler import bind_signal_handlers, StopHandler +from main import server_process_main + + +# +# Windows-specific service implementation. This class will be instantiated directly by pythonservice.exe. +# +class AMHostMonitoringService(AmbariService): + AmbariService._svc_name_ = "AmbariMetricsHostMonitoring" + AmbariService._svc_display_name_ = "Ambari Metrics Host Monitoring" + AmbariService._svc_description_ = "Ambari Metrics Host Monitoring Service" + + AmbariService._AdjustServiceVersion() + + # Adds the necessary script dir to the Python's modules path. + # Modify this as the deployed product's dir structure changes. + def _adjustPythonPath(self, current_dir): + python_path = os.path.join(current_dir, "sbin") + sys.path.insert(0, python_path) + pass + + def SvcDoRun(self): + scmStatus = SvcStatusCallback(self) + + self.redirect_output_streams() + + stopHandler = StopHandler(AMHostMonitoringService._heventSvcStop) + bind_signal_handlers(stopHandler) + + AMHostMonitoringService.set_ctrl_c_handler(ctrlHandler) + + server_process_main(stopHandler, scmStatus) + pass + + def _InitOptionsParser(self): + return init_options_parser() + + def redirect_output_streams(self): + self._RedirectOutputStreamsToFile(SERVER_OUT_FILE) + pass + + +def ctrlHandler(ctrlType): + AMHostMonitoringService.DefCtrlCHandler() + return True + + +def svcsetup(): + AMHostMonitoringService.set_ctrl_c_handler(ctrlHandler) + # we don't save password between 'setup' runs, so we can't run Install every time. We run 'setup' only if user and + # password provided or if service not installed + if (SERVICE_USERNAME_KEY in os.environ and SERVICE_PASSWORD_KEY in os.environ): + AMHostMonitoringService.Install(username=os.environ[SERVICE_USERNAME_KEY], password=os.environ[SERVICE_PASSWORD_KEY]) + elif AMHostMonitoringService.QueryStatus() == "not installed": + AMHostMonitoringService.Install() + pass + + +# +# Starts the Ambari Metrics Collector. The server can start as a service or standalone process. +# args: +# options.is_process = True - start the server as a process. For now, there is no restrictions for the number of +# server instances that can run like this. +# options.is_process = False - start the server in normal mode, as a Windows service. If the Ambari Metrics Collector +# is not registered as a service, the function fails. By default, only one instance of the service can +# possibly run. +# +def start(options): + AMHostMonitoringService.set_ctrl_c_handler(ctrlHandler) + + if options.is_process: + #Run as a normal process. Invoke the ServiceMain directly. + stopHandler = StopHandler(AMHostMonitoringService._heventSvcStop) + bind_signal_handlers(stopHandler) + server_process_main(stopHandler) + else: + AMHostMonitoringService.Start() + +# +# Stops the Ambari Metrics Collector. Ineffective when the server is started as a standalone process. +# +def stop(): + AMHostMonitoringService.Stop() + +# +# Prints the Ambari Metrics Collector service status. +# +def svcstatus(options): + options.exit_message = None + + statusStr = AMHostMonitoringService.QueryStatus() + print "Ambari Metrics Collector is " + statusStr + + +def init_options_parser(): + parser = optparse.OptionParser(usage="usage: %prog action [options]", ) + parser.add_option('-d', '--debug', action="store_true", dest='debug', default=False, + help="Start Ambari Metrics Host Monitoring in debug mode") + parser.add_option('-p', '--process', action="store_true", dest='is_process', default=False, + help="Start Ambari Metrics Host Monitoring as a normal process, not as a service") + + # --help reserved for help + return parser + +def find_python_exe_path(): + paths = "." + os.pathsep + os.environ["PATH"] + + # Find python.exe by attempting to load it as a resource dll + python_path = search_file("python.exe", paths) + return os.path.dirname(python_path) + + +def find_psutil_egg(): + abs_subdir = os.path.join(os.getcwd(), "sbin", "psutil", "build") + egg_files = glob.glob(os.path.join(abs_subdir, "psutil*-win-amd64.egg")) + if egg_files is None or len(egg_files) == 0: + err = "Unable to find the expected psutil egg file in {0}. " \ + "Verify that the installation carried out correctly.".format(abs_subdir) + raise FatalException(1, err) + if len(egg_files) > 1: + err = "Multiple psutil egg files found in {0}".format(abs_subdir) + print_warning_msg(err) + #Return the latest + return egg_files[len(egg_files) - 1] + + +def setup_psutil(): + python_exe_path = find_python_exe_path() + egg_file = find_psutil_egg() + cmd = [os.path.join(python_exe_path, "Scripts", "easy_install"), "--upgrade", egg_file] + + retval, std_out, std_err = run_os_command(cmd) + if std_err is not None and std_err != '': + print_warning_msg(std_err) + print_info_msg(std_out) + if 0 != retval: + err = "Psutil egg installation failed. Exit code={0}, err output={1}".format(retval, std_err) + raise FatalException(1, err) + + # Now the egg should be installed in the site_packages dir + +def win_main(): + parser = init_options_parser() + (options, args) = parser.parse_args() + + options.warnings = [] + options.exit_message = None + + if options.debug: + sys.frozen = 'windows_exe' # Fake py2exe so we can debug + + if len(args) == 0: + print parser.print_help() + parser.error("No action entered") + + action = args[0] + + try: + if action == SETUP_ACTION: + setup_psutil() + svcsetup() + elif action == START_ACTION: + start(options) + elif action == STOP_ACTION: + stop() + elif action == RESTART_ACTION: + stop() + start(options) + elif action == STATUS_ACTION: + svcstatus(options) + else: + parser.error("Invalid action") + + if options.warnings: + for warning in options.warnings: + print_warning_msg(warning) + pass + options.exit_message = "Ambari Metrics Host Monitoring '%s' completed with warnings." % action + pass + except FatalException as e: + if e.reason is not None: + print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(e.code, e.reason)) + sys.exit(e.code) + except NonFatalException as e: + options.exit_message = "Ambari Metrics Host Monitoring '%s' completed with warnings." % action + if e.reason is not None: + print_warning_msg(e.reason) + + if options.exit_message is not None: + print options.exit_message + + sys.exit(0) + +if __name__ == "__main__": + try: + win_main() + except (KeyboardInterrupt, EOFError): + print("\nAborting ... Keyboard Interrupt.") + sys.exit(1) http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py index 996120f..15ad117 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/__init__.py @@ -19,8 +19,9 @@ limitations under the License. """ import os, sys + path = os.path.abspath(__file__) -path = os.path.join(os.path.dirname(os.path.dirname(path)), "psutil/build/") +path = os.path.normpath(os.path.join(os.path.dirname(path), "psutil", "build")) for dir in os.walk(path).next()[1]: if 'lib' in dir: http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py index daabf37..463c98c 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py @@ -22,10 +22,67 @@ import ConfigParser import StringIO import json import os +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyImpl + + +# +# Abstraction for OS-dependent configuration defaults +# +class ConfigDefaults(object): + def get_config_file_path(self): + pass + def get_metric_file_path(self): + pass + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class ConfigDefaultsWindows(ConfigDefaults): + def __init__(self): + self._CONFIG_FILE_PATH = "conf\\metric_monitor.ini" + self._METRIC_FILE_PATH = "conf\\metric_groups.conf" + pass + + def get_config_file_path(self): + return self._CONFIG_FILE_PATH + def get_metric_file_path(self): + return self._METRIC_FILE_PATH + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class ConfigDefaultsLinux(ConfigDefaults): + def __init__(self): + self._CONFIG_FILE_PATH = "/etc/ambari-metrics-monitor/conf/metric_monitor.ini" + self._METRIC_FILE_PATH = "/etc/ambari-metrics-monitor/conf/metric_groups.conf" + pass + + def get_config_file_path(self): + return self._CONFIG_FILE_PATH + def get_metric_file_path(self): + return self._METRIC_FILE_PATH + +configDefaults = ConfigDefaults() + config = ConfigParser.RawConfigParser() -CONFIG_FILE_PATH = "/etc/ambari-metrics-monitor/conf/metric_monitor.ini" -METRIC_FILE_PATH = "/etc/ambari-metrics-monitor/conf/metric_groups.conf" + +CONFIG_FILE_PATH = configDefaults.get_config_file_path() +METRIC_FILE_PATH = configDefaults.get_metric_file_path() + +OUT_DIR = os.path.join(os.sep, "var", "log", "ambari-metrics-host-monitoring") +SERVER_OUT_FILE = OUT_DIR + os.sep + "ambari-metrics-host-monitoring.out" +SERVER_LOG_FILE = OUT_DIR + os.sep + "ambari-metrics-host-monitoring.log" + +PID_DIR = os.path.join(os.sep, "var", "run", "ambari-metrics-host-monitoring") +PID_OUT_FILE = PID_DIR + os.sep + "ambari-metrics-host-monitoring.pid" +EXITCODE_OUT_FILE = PID_DIR + os.sep + "ambari-metrics-host-monitoring.exitcode" + +SERVICE_USERNAME_KEY = "TMP_AMHM_USERNAME" +SERVICE_PASSWORD_KEY = "TMP_AMHM_PASSWORD" + +SETUP_ACTION = "setup" +START_ACTION = "start" +STOP_ACTION = "stop" +RESTART_ACTION = "restart" +STATUS_ACTION = "status" config_content = """ [default] @@ -96,6 +153,11 @@ class Configuration: self.metric_groups = json.load(open(METRIC_FILE_PATH)) else: print 'No metric configs found at {0}'.format(METRIC_FILE_PATH) + self.metric_groups = \ + { + 'host_metric_groups': [], + 'process_metric_groups': [] + } pass def getConfig(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py index 51f0980..e0ef804 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py @@ -33,7 +33,7 @@ logger = logging.getLogger() class Controller(threading.Thread): - def __init__(self, config): + def __init__(self, config, stop_handler): # Process initialization code threading.Thread.__init__(self) logger.debug('Initializing Controller thread.') @@ -48,27 +48,42 @@ class Controller(threading.Thread): self.metric_collector = MetricsCollector(self.event_queue, self.application_metric_map) self.server_url = config.get_server_address() self.sleep_interval = config.get_collector_sleep_interval() + self._stop_handler = stop_handler self.initialize_events_cache() - self.emitter = Emitter(self.config, self.application_metric_map) + self.emitter = Emitter(self.config, self.application_metric_map, stop_handler) + self._t = None def run(self): logger.info('Running Controller thread: %s' % threading.currentThread().getName()) - # Wake every 5 seconds to push events to the queue + + self.start_emitter() + + # Wake every 5 seconds to push events to the queue while True: if (self.event_queue.full()): logger.warn('Event Queue full!! Suspending further collections.') else: self.enqueque_events() pass - time.sleep(self.sleep_interval) + #Wait for the service stop event instead of sleeping blindly + if 0 == self._stop_handler.wait(self.sleep_interval): + logger.info('Shutting down Controller thread') + break + + if not self._t is None: + self._t.cancel() + self._t.join(5) + + #The emitter thread should have stopped by now, just ensure it has shut down properly + self.emitter.join(5) pass # TODO: Optimize to not use Timer class and use the Queue instead def enqueque_events(self): # Queue events for up to a minute for event in self.events_cache: - t = Timer(event.get_collect_interval(), self.metric_collector.process_event, args=(event,)) - t.start() + self._t = Timer(event.get_collect_interval(), self.metric_collector.process_event, args=(event,)) + self._t.start() pass def initialize_events_cache(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py index be83250..c3fd543 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py @@ -32,12 +32,13 @@ class Emitter(threading.Thread): """ Wake up every send interval seconds and empty the application metric map. """ - def __init__(self, config, application_metric_map): + def __init__(self, config, application_metric_map, stop_handler): threading.Thread.__init__(self) logger.debug('Initializing Emitter thread.') self.lock = threading.Lock() self.collector_address = config.get_server_address() self.send_interval = config.get_send_interval() + self._stop_handler = stop_handler self.application_metric_map = application_metric_map def run(self): @@ -45,10 +46,16 @@ class Emitter(threading.Thread): while True: try: self.submit_metrics() - time.sleep(self.send_interval) + #Wait for the service stop event instead of sleeping blindly + if 0 == self._stop_handler.wait(self.send_interval): + logger.info('Shutting down Emitter thread') + return except Exception, e: logger.warn('Unable to emit events. %s' % str(e)) - time.sleep(self.RETRY_SLEEP_INTERVAL) + #Wait for the service stop event instead of sleeping blindly + if 0 == self._stop_handler.wait(self.RETRY_SLEEP_INTERVAL): + logger.info('Shutting down Emitter thread - abort retry') + return logger.info('Retrying emit after %s seconds.' % self.RETRY_SLEEP_INTERVAL) pass @@ -69,7 +76,9 @@ class Emitter(threading.Thread): logger.warn("Error sending metrics to server. Retrying after {0} " "...".format(self.RETRY_SLEEP_INTERVAL)) retry_count += 1 - time.sleep(self.RETRY_SLEEP_INTERVAL) + #Wait for the service stop event instead of sleeping blindly + if 0 == self._stop_handler.wait(self.RETRY_SLEEP_INTERVAL): + return pass pass http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py new file mode 100644 index 0000000..bfb6957 --- /dev/null +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import logging +import os +import signal +import threading +import traceback + +from ambari_commons import OSConst, OSCheck +from ambari_commons.exceptions import FatalException +from ambari_commons.os_family_impl import OsFamilyImpl + + +logger = logging.getLogger() + +_handler = None + + +class StopHandler(object): + def set_stop(self): + pass + + def wait(self, timeout=None): + return -1 + + +# +# Windows implementation +# +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class StopHandlerWindows(StopHandler): + def __init__(self, stopEvent=None): + import win32event + # Event used to gracefully stop the process + if stopEvent is None: + # Allow standalone testing + self._heventStop = win32event.CreateEvent(None, 0, 0, None) + else: + # Allow one unique event per process + self._heventStop = stopEvent + + def set_stop(self): + import win32event + win32event.SetEvent(self._heventStop) + + def wait(self, timeout=None): + ''' + :param timeout: Time to wait, in seconds. + :return: 0 == stop event signaled, -1 = timeout + ''' + import win32event + + if timeout is None: + timeout = win32event.INFINITE + else: + timeout = timeout * 1000 + + result = win32event.WaitForSingleObject(self._heventStop, timeout) + if(win32event.WAIT_OBJECT_0 != result and win32event.WAIT_TIMEOUT != result): + raise FatalException(-1, "Error waiting for stop event: " + str(result)) + if (win32event.WAIT_TIMEOUT == result): + return -1 + logger.info("Stop event received") + return result # 0 -> stop + + +# +# Linux implementation +# +def signal_handler(signum, frame): + global _handler + _handler.set_stop() + +def debug(sig, frame): + """Interrupt running process, and provide a python prompt for + interactive debugging.""" + d = {'_frame': frame} # Allow access to frame object. + d.update(frame.f_globals) # Unless shadowed by global + d.update(frame.f_locals) + + message = "Signal received : entering python shell.\nTraceback:\n" + message += ''.join(traceback.format_stack(frame)) + logger.info(message) + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class StopHandlerLinux(StopHandler): + def __init__(self, stopEvent=None): + # Event used to gracefully stop the process + if stopEvent is None: + # Allow standalone testing + self.stop_event = threading.Event() + else: + # Allow one unique event per process + self.stop_event = stopEvent + + def set_stop(self): + self.stop_event.set() + + def wait(self, timeout=None): + # Stop process when stop event received + if self.stop_event.wait(timeout): + logger.info("Stop event received") + return 0 + # Timeout + return -1 + + +def bind_signal_handlers(new_handler=None): + if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY: + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGUSR1, debug) + + if new_handler is None: + global _handler + _handler = StopHandler() + else: + _handler = new_handler + return _handler http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py index 09ae7e4..37e77e5 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/main.py @@ -18,27 +18,67 @@ See the License for the specific language governing permissions and limitations under the License. ''' -import core -from core.controller import Controller -from core.config_reader import Configuration import logging -import signal +import os import sys +from ambari_commons.os_utils import remove_file + +from core.controller import Controller +from core.config_reader import Configuration, PID_OUT_FILE, SERVER_LOG_FILE, SERVER_OUT_FILE +from core.stop_handler import bind_signal_handlers + + logger = logging.getLogger() + +def save_pid(pid, pidfile): + """ + Save pid to pidfile. + """ + try: + pfile = open(pidfile, "w") + pfile.write("%s\n" % pid) + except IOError: + pass + finally: + try: + pfile.close() + except: + pass + + def main(argv=None): # Allow Ctrl-C - signal.signal(signal.SIGINT, signal.SIG_DFL) + stop_handler = bind_signal_handlers() + + server_process_main(stop_handler) + +def server_process_main(stop_handler, scmStatus=None): + if scmStatus is not None: + scmStatus.reportStartPending() + + save_pid(os.getpid(), PID_OUT_FILE) config = Configuration() - controller = Controller(config) - + controller = Controller(config, stop_handler) + _init_logging(config) - + logger.info('Starting Server RPC Thread: %s' % ' '.join(sys.argv)) controller.start() - controller.start_emitter() + + print "Server out at: " + SERVER_OUT_FILE + print "Server log at: " + SERVER_LOG_FILE + + if scmStatus is not None: + scmStatus.reportStarted() + + #The controller thread finishes when the stop event is signaled + controller.join() + + remove_file(PID_OUT_FILE) + pass def _init_logging(config): _levels = { http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py index 05362bf..56e6475 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestEmitter.py @@ -17,48 +17,63 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' + +import json +import urllib2 + import logging from unittest import TestCase - -from application_metric_map import ApplicationMetricMap -from config_reader import Configuration -from emitter import Emitter +from only_for_platform import get_platform, PLATFORM_WINDOWS from mock.mock import patch, MagicMock -import json -import urllib2 +if get_platform() != PLATFORM_WINDOWS: + os_distro_value = ('Suse','11','Final') +else: + os_distro_value = ('win2012serverr2','6.3','WindowsServer') + +with patch("platform.linux_distribution", return_value = os_distro_value): + from ambari_commons import OSCheck + from application_metric_map import ApplicationMetricMap + from config_reader import Configuration + from emitter import Emitter + from stop_handler import bind_signal_handlers logger = logging.getLogger() class TestEmitter(TestCase): - + + @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) @patch("urllib2.urlopen") def testJavaHomeAvailableCheck(self, url_open_mock): url_open_mock.return_value = MagicMock() url_open_mock.return_value.getcode.return_value = 200 self.assertEqual(urllib2.urlopen(None, None).getcode(), 200) url_open_mock.reset_mock() - + + stop_handler = bind_signal_handlers() + config = Configuration() application_metric_map = ApplicationMetricMap("host","10.10.10.10") application_metric_map.clear() application_metric_map.put_metric("APP1", {"metric1":1}, 1) - emitter = Emitter(config, application_metric_map) + emitter = Emitter(config, application_metric_map, stop_handler) emitter.submit_metrics() self.assertEqual(url_open_mock.call_count, 1) self.assertUrlData(url_open_mock) - - + + + @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) @patch("urllib2.urlopen") def testRetryFetch(self, url_open_mock): - + stop_handler = bind_signal_handlers() + config = Configuration() application_metric_map = ApplicationMetricMap("host","10.10.10.10") application_metric_map.clear() application_metric_map.put_metric("APP1", {"metric1":1}, 1) - emitter = Emitter(config, application_metric_map) + emitter = Emitter(config, application_metric_map, stop_handler) emitter.RETRY_SLEEP_INTERVAL = .001 emitter.submit_metrics() http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ambari-metrics-collector.cmd ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ambari-metrics-collector.cmd b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ambari-metrics-collector.cmd new file mode 100644 index 0000000..2cfff6f --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ambari-metrics-collector.cmd @@ -0,0 +1,17 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@echo off +python.exe -u %~dp0\sbin\main.py %* 2>&1 http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-env.cmd ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-env.cmd b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-env.cmd new file mode 100644 index 0000000..af99fc4 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-env.cmd @@ -0,0 +1,16 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem Set environment variables here. http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-site.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-site.xml b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-site.xml new file mode 100644 index 0000000..c2dd100 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams-site.xml @@ -0,0 +1,25 @@ +<?xml version="1.0"?> + +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration> + + <!-- Site specific AMS configuration properties --> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams.properties ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams.properties b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams.properties new file mode 100644 index 0000000..3df60b0 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/ams.properties @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-timelineservice/conf/windows/log4j.properties ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/log4j.properties b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/log4j.properties new file mode 100644 index 0000000..d354a27 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/log4j.properties @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Define some default values that can be overridden by system properties +# Root logger option +log4j.rootLogger=INFO,file + +# Direct log messages to a log file +log4j.appender.file=org.apache.log4j.RollingFileAppender +log4j.appender.file.File=\\var\\log\\ambari-metrics-collector\\ambari-metrics-collector.log +log4j.appender.file.MaxFileSize=80MB +log4j.appender.file.MaxBackupIndex=60 +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p [%t] %c{1}:%L - %m%n http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-timelineservice/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml index 196d7a1..46504a7 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml +++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml @@ -42,6 +42,33 @@ <build> <plugins> <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.8</version> + <executions> + <execution> + <id>parse-version</id> + <phase>validate</phase> + <goals> + <goal>parse-version</goal> + </goals> + </execution> + <execution> + <id>regex-property</id> + <goals> + <goal>regex-property</goal> + </goals> + <configuration> + <name>ambariVersion</name> + <value>${project.version}</value> + <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex> + <replacement>$1.$2.$3</replacement> + <failIfNoMatch>false</failIfNoMatch> + </configuration> + </execution> + </executions> + </plugin> + <plugin> <artifactId>maven-dependency-plugin</artifactId> <executions> <execution> @@ -59,6 +86,10 @@ </executions> </plugin> <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.0</version> + </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-antrun-plugin</artifactId> <version>1.7</version> http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/__init__.py b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/__init__.py new file mode 100644 index 0000000..38e1a5e --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" http://git-wip-us.apache.org/repos/asf/ambari/blob/6c21b094/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/properties.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/properties.py b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/properties.py new file mode 100644 index 0000000..8e00762 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/python/ambari_metrics_collector/properties.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import os +import re +import time + +#Apache License Header +ASF_LICENSE_HEADER = ''' +# Copyright 2011 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' + +# A Python replacement for java.util.Properties +# Based on http://code.activestate.com/recipes +# /496795-a-python-replacement-for-javautilproperties/ +class Properties(object): + def __init__(self, props=None): + self._props = {} + self._origprops = {} + self._keymap = {} + + self.othercharre = re.compile(r'(?<!\\)(\s*\=)|(?<!\\)(\s*\:)') + self.othercharre2 = re.compile(r'(\s*\=)|(\s*\:)') + self.bspacere = re.compile(r'\\(?!\s$)') + + def __parse(self, lines): + lineno = 0 + i = iter(lines) + for line in i: + lineno += 1 + line = line.strip() + if not line: + continue + if line[0] == '#': + continue + escaped = False + sepidx = -1 + flag = 0 + m = self.othercharre.search(line) + if m: + first, last = m.span() + start, end = 0, first + flag = 1 + wspacere = re.compile(r'(?<![\\\=\:])(\s)') + else: + if self.othercharre2.search(line): + wspacere = re.compile(r'(?<![\\])(\s)') + start, end = 0, len(line) + m2 = wspacere.search(line, start, end) + if m2: + first, last = m2.span() + sepidx = first + elif m: + first, last = m.span() + sepidx = last - 1 + while line[-1] == '\\': + nextline = i.next() + nextline = nextline.strip() + lineno += 1 + line = line[:-1] + nextline + if sepidx != -1: + key, value = line[:sepidx], line[sepidx + 1:] + else: + key, value = line, '' + self.process_pair(key, value) + + def process_pair(self, key, value): + oldkey = key + oldvalue = value + keyparts = self.bspacere.split(key) + strippable = False + lastpart = keyparts[-1] + if lastpart.find('\\ ') != -1: + keyparts[-1] = lastpart.replace('\\', '') + elif lastpart and lastpart[-1] == ' ': + strippable = True + key = ''.join(keyparts) + if strippable: + key = key.strip() + oldkey = oldkey.strip() + oldvalue = self.unescape(oldvalue) + value = self.unescape(value) + self._props[key] = None if value is None else value.strip() + if self._keymap.has_key(key): + oldkey = self._keymap.get(key) + self._origprops[oldkey] = None if oldvalue is None else oldvalue.strip() + else: + self._origprops[oldkey] = None if oldvalue is None else oldvalue.strip() + self._keymap[key] = oldkey + + def unescape(self, value): + newvalue = value + if not value is None: + newvalue = value.replace('\:', ':') + newvalue = newvalue.replace('\=', '=') + return newvalue + + def removeOldProp(self, key): + if self._origprops.has_key(key): + del self._origprops[key] + pass + + def removeProp(self, key): + if self._props.has_key(key): + del self._props[key] + pass + + def load(self, stream): + if type(stream) is not file: + raise TypeError, 'Argument should be a file object!' + if stream.mode != 'r': + raise ValueError, 'Stream should be opened in read-only mode!' + try: + self.fileName = os.path.abspath(stream.name) + lines = stream.readlines() + self.__parse(lines) + except IOError: + raise + + def get_property(self, key): + return self._props.get(key, '') + + def propertyNames(self): + return self._props.keys() + + def getPropertyDict(self): + return self._props + + def __getitem__(self, name): + return self.get_property(name) + + def __getattr__(self, name): + try: + return self.__dict__[name] + except KeyError: + if hasattr(self._props, name): + return getattr(self._props, name) + + def sort_props(self): + tmp_props = {} + for key in sorted(self._props.iterkeys()): + tmp_props[key] = self._props[key] + self._props = tmp_props + pass + + def sort_origprops(self): + tmp_props = self._origprops.copy() + self._origprops.clear() + for key in sorted(tmp_props.iterkeys()): + self._origprops[key] = tmp_props[key] + pass + + def store(self, out, header=""): + """ Write the properties list to the stream 'out' along + with the optional 'header' """ + if out.mode[0] != 'w': + raise ValueError, 'Steam should be opened in write mode!' + try: + out.write(''.join(('#', ASF_LICENSE_HEADER, '\n'))) + out.write(''.join(('#', header, '\n'))) + # Write timestamp + tstamp = time.strftime('%a %b %d %H:%M:%S %Z %Y', time.localtime()) + out.write(''.join(('#', tstamp, '\n'))) + # Write properties from the pristine dictionary + for prop, val in self._origprops.items(): + if val is not None: + out.write(''.join((prop, '=', val, '\n'))) + out.close() + except IOError: + raise + + def store_ordered(self, out, header=""): + """ Write the properties list to the stream 'out' along + with the optional 'header' """ + if out.mode[0] != 'w': + raise ValueError, 'Steam should be opened in write mode!' + try: + out.write(''.join(('#', ASF_LICENSE_HEADER, '\n'))) + out.write(''.join(('#', header, '\n'))) + # Write timestamp + tstamp = time.strftime('%a %b %d %H:%M:%S %Z %Y', time.localtime()) + out.write(''.join(('#', tstamp, '\n'))) + # Write properties from the pristine dictionary + for key in sorted(self._origprops.iterkeys()): + val = self._origprops[key] + if val is not None: + out.write(''.join((key, '=', val, '\n'))) + out.close() + except IOError: + raise
