AMBARI-18854. Add ability to add custom input descriptor to logfeeder (Miklos Gergely via oleewere)
Change-Id: I659c5fbc94af8ba54a26698e5b1ebe32fedeb874 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dc76e4e1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dc76e4e1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dc76e4e1 Branch: refs/heads/branch-2.5 Commit: dc76e4e120c5d4d8c32547eeec1fccb99015971b Parents: 02e6251 Author: Miklos Gergely <[email protected]> Authored: Wed Nov 16 11:47:39 2016 +0100 Committer: oleewere <[email protected]> Committed: Wed Nov 16 12:47:32 2016 +0100 ---------------------------------------------------------------------- .../logfeeder-custom-logsearch-conf.xml | 46 ++++++++++++++++++++ .../scripts/logsearch_config_aggregator.py | 14 +++++- .../LOGSEARCH/0.5.0/package/scripts/params.py | 1 + .../0.5.0/package/scripts/setup_logfeeder.py | 1 + .../templates/HadoopServiceConfig.json.j2 | 14 +++--- .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 3 ++ 6 files changed, 69 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/dc76e4e1/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml new file mode 100644 index 0000000..8358266 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml @@ -0,0 +1,46 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_component_mappings</name> + <display-name>Service Component mapping</display-name> + <description>Logsearch service component logid mapping list (e.g.: SERVICE1=S1_COMPONENT1:logid1,logid2;S1_COMPONENT2:logid3|SERVICE2=...)</description> + <value></value> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dc76e4e1/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py index ad515ee..c14900e 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py @@ -46,7 +46,7 @@ def get_logfeeder_metadata(logsearch_meta_configs): """ logfeeder_contents = {} for key, value in logsearch_meta_configs.iteritems(): - if 'content' in logsearch_meta_configs[key]: + if 'content' in logsearch_meta_configs[key] and logsearch_meta_configs[key]['content'].strip(): logfeeder_contents[key] = logsearch_meta_configs[key]['content'] Logger.info("Found logfeeder pattern content in " + key) return logfeeder_contents @@ -61,7 +61,17 @@ def get_logsearch_metadata(logsearch_meta_configs): if 'service_name' in logsearch_meta_configs[key] and 'component_mappings' in logsearch_meta_configs[key]: service_name = logsearch_meta_configs[key]['service_name'] component_mappings = __parse_component_mappings(logsearch_meta_configs[key]['component_mappings']) - logsearch_service_component_mappings[service_name] = component_mappings + if service_name.strip() and component_mappings: + logsearch_service_component_mappings[service_name] = component_mappings + if 'service_component_mappings' in logsearch_meta_configs[key]: + service_component_mappings = logsearch_meta_configs[key]['service_component_mappings'] + if service_component_mappings.strip(): + for service_component_mapping in service_component_mappings.split('|'): + tokens = service_component_mapping.split('=') + service_name = tokens[0] + component_mappings = __parse_component_mappings(tokens[1]) + if service_name.strip() and component_mappings: + logsearch_service_component_mappings[service_name] = component_mappings return logsearch_service_component_mappings http://git-wip-us.apache.org/repos/asf/ambari/blob/dc76e4e1/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index 1f5064a..bdb0c10 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -147,6 +147,7 @@ ambari_server_log_dir = '/var/log/ambari-server' ambari_agent_log_dir = '/var/log/ambari-agent' hst_log_dir = '/var/log/hst' hst_activity_log_dir = '/var/log/smartsense-activity' +nifi_log_dir = default('/configurations/nifi-env/nifi_node_log_dir', '/var/log/nifi') # System logs logfeeder_system_messages_content = config['configurations']['logfeeder-system_log-env']['logfeeder_system_messages_content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/dc76e4e1/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py index e90d623..021c167 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py @@ -67,6 +67,7 @@ def setup_logfeeder(): content=Template(file_name + ".j2") ) + File(format("{logsearch_logfeeder_conf}/input.config-logfeeder-custom.json"), action='delete') for service, pattern_content in params.logfeeder_metadata.iteritems(): File(format("{logsearch_logfeeder_conf}/input.config-" + service.replace('-logsearch-conf', '') + ".json"), content=InlineTemplate(pattern_content, extra_imports=[default]) http://git-wip-us.apache.org/repos/asf/ambari/blob/dc76e4e1/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 index 7e9153f..efa51fa 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 @@ -18,24 +18,22 @@ { "service": { {% if logsearch_metadata and logsearch_metadata.keys()|length > 0 %} - {% for metadata_key, component_list in logsearch_metadata.iteritems() %} +{% for metadata_key, component_list in logsearch_metadata.iteritems() %} "{{ metadata_key.lower() }}": { "label": "{{ metadata_key }}", "components": [ - {% for component in component_list %} +{% for component in component_list %} { "name": "{{ component }}" - } - {% if not loop.last %} - , - {% endif %} - {% endfor %} + }{% if not loop.last %},{% endif %} + +{% endfor %} ], "dependencies": [ ] } , - {% endfor %} +{% endfor %} {% endif %} {% if logfeeder_system_log_enabled %} "system" : { http://git-wip-us.apache.org/repos/asf/ambari/blob/dc76e4e1/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py index 7a13637..773b75a 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -89,6 +89,9 @@ class TestLogFeeder(RMFTestCase): self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/' + file_name, content=Template(file_name + ".j2") ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/input.config-logfeeder-custom.json', + action=['delete'] + ) self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/input.config-zookeeper.json', content=InlineTemplate("pattern content") )
