This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push: new 5e67052 [AMBARI-23883] Log Feeder: ambari services should be able to generate … (#1309) 5e67052 is described below commit 5e6705292e55f80fea054b341ce6c76adfebc0bb Author: Olivér Szabó <oleew...@gmail.com> AuthorDate: Fri May 18 00:27:20 2018 +0200 [AMBARI-23883] Log Feeder: ambari services should be able to generate … (#1309) * AMBARI-23883. Log Feeder: ambari services should be able to generate logfeeder configs outside of after-INSTALL hook. * AMBARI-23883. Add docs. * AMBARI-23883. Add logsearch + infra-solr input patterns and remove env from method params * AMBARI-23883. Support pattern generation for logfeeder itself. --- .../src/main/python/ambari_commons/constants.py | 2 + .../functions/generate_logfeeder_input_config.py | 44 ++++++++++++++++++++++ .../0.1.0/package/scripts/infra_solr.py | 5 +++ .../0.5.0/package/scripts/setup_logfeeder.py | 2 + .../0.5.0/package/scripts/setup_logsearch.py | 3 ++ .../stack-hooks/after-INSTALL/scripts/params.py | 3 +- .../stacks/2.4/AMBARI_INFRA/test_infra_solr.py | 11 ++++++ .../python/stacks/2.4/LOGSEARCH/test_logfeeder.py | 10 +++++ .../python/stacks/2.4/LOGSEARCH/test_logsearch.py | 13 +++++++ 9 files changed, 92 insertions(+), 1 deletion(-) diff --git a/ambari-common/src/main/python/ambari_commons/constants.py b/ambari-common/src/main/python/ambari_commons/constants.py index f1ab4a2..abb07da 100644 --- a/ambari-common/src/main/python/ambari_commons/constants.py +++ b/ambari-common/src/main/python/ambari_commons/constants.py @@ -26,6 +26,8 @@ UPGRADE_TYPE_HOST_ORDERED = "host_ordered" AGENT_TMP_DIR = "/var/lib/ambari-agent/tmp" +LOGFEEDER_CONF_DIR = "/usr/lib/ambari-logsearch-logfeeder/conf" + class SERVICE: """ Constants for service names to avoid hardcoding strings. diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/generate_logfeeder_input_config.py b/ambari-common/src/main/python/resource_management/libraries/functions/generate_logfeeder_input_config.py new file mode 100755 index 0000000..84ab492 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/generate_logfeeder_input_config.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os +from resource_management.core.logger import Logger +from ambari_commons.constants import LOGFEEDER_CONF_DIR +from resource_management.core.resources import File, Directory + +__all__ = ["generate_logfeeder_input_config"] + +def generate_logfeeder_input_config(type, content): + """ + :param type: type of the logfeeder input config (most likely a service name: hdfs), + it will be generated as input.config-<type>.json in logfeeder config folder + :param content: generated template for the input config json file (you can use Template or InlineTemplate) + """ + import params + Directory(LOGFEEDER_CONF_DIR, + mode=0755, + cd_access='a', + create_parents=True + ) + input_file_name = 'input.config-' + type + '.json' + Logger.info("Generate Log Feeder config file: " + os.path.join(LOGFEEDER_CONF_DIR, input_file_name)) + File(os.path.join(LOGFEEDER_CONF_DIR, input_file_name), + content=content, + mode=0644 + ) \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py index 363f8f4..a11f207 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py @@ -19,13 +19,16 @@ limitations under the License. import sys from resource_management.core.logger import Logger +from resource_management.core.source import Template from resource_management.core.resources.system import Execute, File from resource_management.core.resources.zkmigrator import ZkMigrator from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format from resource_management.libraries.functions.get_user_call_output import get_user_call_output from resource_management.libraries.functions.show_logs import show_logs from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.generate_logfeeder_input_config import generate_logfeeder_input_config from collection import backup_collection, restore_collection from migrate import migrate_index @@ -48,6 +51,8 @@ class InfraSolr(Script): env.set_params(params) self.configure(env) + generate_logfeeder_input_config('ambari-infra', Template("input.config-ambari-infra.json.j2", extra_imports=[default])) + setup_solr_znode_env() start_cmd = format('{solr_bindir}/solr start -cloud -noprompt -s {infra_solr_datadir} -Dsolr.kerberos.name.rules=\'{infra_solr_kerberos_name_rules}\' >> {infra_solr_log} 2>&1') \ if params.security_enabled else format('{solr_bindir}/solr start -cloud -noprompt -s {infra_solr_datadir} >> {infra_solr_log} 2>&1') diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py index 52b1d76..76ec5c5 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py @@ -21,6 +21,7 @@ from resource_management.libraries.functions.default import default from resource_management.core.resources.system import Directory, File from resource_management.libraries.functions.format import format from resource_management.core.source import InlineTemplate, Template +from resource_management.libraries.functions.generate_logfeeder_input_config import generate_logfeeder_input_config from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.libraries.functions.security_commons import update_credential_provider_path, HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME @@ -121,6 +122,7 @@ def setup_logfeeder(): content=params.logfeeder_secure_log_content ) + generate_logfeeder_input_config('logsearch', Template("input.config-logsearch.json.j2", extra_imports=[default])) if params.logsearch_solr_kerberos_enabled: File(format("{logfeeder_jaas_file}"), diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py index 63b33e7..c6ad498 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py @@ -19,10 +19,12 @@ limitations under the License. from resource_management.core.exceptions import Fail from resource_management.core.resources.system import Directory, Execute, File +from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format from resource_management.core.source import InlineTemplate, Template from resource_management.libraries.functions import solr_cloud_util from resource_management.libraries.functions.decorator import retry +from resource_management.libraries.functions.generate_logfeeder_input_config import generate_logfeeder_input_config from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.libraries.functions.security_commons import update_credential_provider_path, HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME @@ -137,6 +139,7 @@ def setup_logsearch(): Execute(("chmod", "-R", "ugo+r", format("{logsearch_server_conf}/solr_configsets")), sudo=True ) + generate_logfeeder_input_config('logsearch', Template("input.config-logsearch.json.j2", extra_imports=[default])) check_znode() if params.security_enabled and not params.logsearch_use_external_solr: diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py index ceec93f..6fc8f1e 100644 --- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py @@ -20,6 +20,7 @@ limitations under the License. import os from ambari_commons.constants import AMBARI_SUDO_BINARY +from ambari_commons.constants import LOGFEEDER_CONF_DIR from resource_management.libraries.script import Script from resource_management.libraries.script.script import get_config_lock_file from resource_management.libraries.functions import default @@ -47,7 +48,7 @@ major_stack_version = get_major_version(stack_version_formatted) service_name = config['serviceName'] # logsearch configuration -logsearch_logfeeder_conf = "/usr/lib/ambari-logsearch-logfeeder/conf" +logsearch_logfeeder_conf = LOGFEEDER_CONF_DIR agent_cache_dir = config['agentLevelParams']['agentCacheDir'] service_package_folder = config['serviceLevelParams']['service_package_folder'] diff --git a/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py b/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py index 237466a..694872c 100644 --- a/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py +++ b/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py @@ -20,6 +20,7 @@ limitations under the License. from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.libraries.functions.default import default from resource_management.libraries.script.config_dictionary import UnknownConfiguration from mock.mock import MagicMock, call, patch @@ -109,6 +110,16 @@ class TestInfraSolr(RMFTestCase): group='root', content = Template('infra-solr.conf.j2') ) + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/input.config-logsearch.json', + mode=0644, + content = Template('input.config-logsearch.json.j2', extra_imports=[default]) + ) self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --create-znode --retry 30 --interval 5') self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --remove-admin-handlers --collection hadoop_logs --retry 5 --interval 10') diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py index 3bc48ac..8ae0acd 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -21,6 +21,7 @@ limitations under the License. import grp from mock.mock import MagicMock, patch from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate +from resource_management.libraries.functions.default import default class TestLogFeeder(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package" @@ -104,6 +105,15 @@ class TestLogFeeder(RMFTestCase): content=InlineTemplate('output-grok-filter'), encoding='utf-8' ) + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/input.config-logsearch.json', + mode=0644, + content = Template('input.config-logsearch.json.j2', extra_imports=[default]) + ) def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py", diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py index 89bc492..0c3d38c 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py @@ -19,6 +19,7 @@ limitations under the License. ''' from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate +from resource_management.libraries.functions.default import default class TestLogSearch(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package" @@ -149,6 +150,18 @@ class TestLogSearch(RMFTestCase): self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/usr/lib/ambari-logsearch-portal/conf/solr_configsets'), sudo = True ) + + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf', + create_parents = True, + cd_access = 'a', + mode = 0755 + ) + + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/input.config-logsearch.json', + mode=0644, + content = Template('input.config-logsearch.json.j2', extra_imports=[default]) + ) + self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 30 --interval 5') -- To stop receiving notification emails like this one, please contact oleew...@apache.org.