This is an automated email from the ASF dual-hosted git repository. jluniya pushed a commit to branch branch-feature-AMBARI-14714 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push: new a33db89 Ambari-23136: Create an interface to read command.json properties via API instead of direct access (#709) a33db89 is described below commit a33db89f0a311b2bcf5c046b6209ccd7dcae3a9d Author: sduan <sd...@hortonworks.com> AuthorDate: Fri Mar 23 08:17:22 2018 -0700 Ambari-23136: Create an interface to read command.json properties via API instead of direct access (#709) * AMBARI-23136: Create an interface to read command.json properties via API instead of direct access * AMBARI-23136: Create an interface to read command.json properties via API instead of direct access --- .../resource_management/libraries/__init__.py | 1 + .../libraries/{ => execution_command}/__init__.py | 6 -- .../execution_command/execution_command.py | 85 ++++++++++++++++++++ .../module_configs.py} | 27 +++++-- .../resource_management/libraries/script/script.py | 22 +++++ .../src/test/python/TestExecutionCommand.py | 60 ++++++++++++++ ambari-common/src/test/python/command.json | 93 ++++++++++++++++++++++ 7 files changed, 282 insertions(+), 12 deletions(-) diff --git a/ambari-common/src/main/python/resource_management/libraries/__init__.py b/ambari-common/src/main/python/resource_management/libraries/__init__.py index b1d3a36..b36d4a2 100644 --- a/ambari-common/src/main/python/resource_management/libraries/__init__.py +++ b/ambari-common/src/main/python/resource_management/libraries/__init__.py @@ -20,6 +20,7 @@ Ambari Agent """ +from resource_management.libraries.execution_command import * from resource_management.libraries.functions import * from resource_management.libraries.resources import * from resource_management.libraries.providers import * diff --git a/ambari-common/src/main/python/resource_management/libraries/__init__.py b/ambari-common/src/main/python/resource_management/libraries/execution_command/__init__.py similarity index 77% copy from ambari-common/src/main/python/resource_management/libraries/__init__.py copy to ambari-common/src/main/python/resource_management/libraries/execution_command/__init__.py index b1d3a36..d05d43b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/__init__.py +++ b/ambari-common/src/main/python/resource_management/libraries/execution_command/__init__.py @@ -16,11 +16,5 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -Ambari Agent - """ -from resource_management.libraries.functions import * -from resource_management.libraries.resources import * -from resource_management.libraries.providers import * -from resource_management.libraries.script import * diff --git a/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py b/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py new file mode 100644 index 0000000..0e3f8e4 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +__all__ = ["ExecutionCommand"] + +import ambari_simplejson + +from resource_management.libraries.execution_command import module_configs + + +class ExecutionCommand(object): + """ + The class maps to a command.json. All command related info should be retrieved from this class object + """ + + def __init__(self, command): + """ + __execution_command is an internal dot access dict object maps to command.json + :param command: json string or a python dict object + """ + self.__execution_command = command + self.__module_configs = module_configs.ModuleConfigs(self.__get_value("configurations")) + + def __get_value(self, key, default_value=None): + """ + A private method to query value with the full path of key, if key does not exist, return default_value + :param key: + :param default_value: + :return: + """ + sub_keys = key.split('/') + value = self.__execution_command + try: + for sub_key in sub_keys: + value = value[sub_key] + return value + except: + return default_value + + def get_module_name(self): + return self.__get_value("serviceName") + + def get_stack_name(self): + return self.__get_value("hostLevelParams/stack_name") + + def get_stack_version(self): + return self.__get_value("hostLevelParams/stack_version") + + def get_new_stack_version_for_upgrade(self): + """ + New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade + :return: + """ + return self.__get_value("commandParams/version") + + def get_host_name(self): + return self.__get_value("hostname") + + def get_java_home(self): + return self.__get_value("hostLevelParams/java_home") + + def get_java_version(self): + java_version = self.__get_value("hostLevelParams/java_version") + return int(java_version) if java_version else None + + def get_module_configs(self): + return self.__module_configs + diff --git a/ambari-common/src/main/python/resource_management/libraries/__init__.py b/ambari-common/src/main/python/resource_management/libraries/execution_command/module_configs.py similarity index 55% copy from ambari-common/src/main/python/resource_management/libraries/__init__.py copy to ambari-common/src/main/python/resource_management/libraries/execution_command/module_configs.py index b1d3a36..967449f 100644 --- a/ambari-common/src/main/python/resource_management/libraries/__init__.py +++ b/ambari-common/src/main/python/resource_management/libraries/execution_command/module_configs.py @@ -16,11 +16,26 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -Ambari Agent - """ -from resource_management.libraries.functions import * -from resource_management.libraries.resources import * -from resource_management.libraries.providers import * -from resource_management.libraries.script import * +__all__ = ["ModuleConfigs"] + + +class ModuleConfigs(object): + """ + This class maps to "/configurations" block in command.json which includes configuration information of a service + """ + + def __init__(self, config): + self.__module_configs = config + + def get_properties(self, module_name, config_type, property_names): + return map(lambda property_name: self.get_property_value(module_name, config_type, property_name), property_names) + + def get_property_value(self, module_name, config_type, property_name, default=None): + try: + return self.__module_configs[config_type][property_name] + except: + return default + + diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index cd528c1..878c83a 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -64,6 +64,8 @@ from resource_management.core.providers import get_provider from resource_management.libraries.functions.fcntl_based_process_lock import FcntlBasedProcessLock from resource_management.libraries.functions.config_helper import get_mpack_name, get_mpack_version, \ get_mpack_instance_name, get_module_name, get_component_type, get_component_instance_name +from resource_management.libraries.execution_command.execution_command import ExecutionCommand +from resource_management.libraries.execution_command.module_configs import ModuleConfigs import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. @@ -146,6 +148,8 @@ class Script(object): 4 path to file with structured command output (file will be created) """ config = None + execution_command = None + module_configs = None stack_version_from_distro_select = None structuredOut = {} command_data_file = "" @@ -339,6 +343,8 @@ class Script(object): with open(self.command_data_file) as f: pass Script.config = ConfigDictionary(json.load(f)) + Script.execution_command = ExecutionCommand(Script.config) + Script.module_configs = Script.execution_command.get_module_configs() # load passwords here(used on windows to impersonate different users) Script.passwords = {} for k, v in _PASSWORD_MAP.iteritems(): @@ -598,6 +604,22 @@ class Script(object): return Script.config @staticmethod + def get_execution_command(): + """ + The dot access dict object holds command.json + :return: + """ + return Script.execution_command + + @staticmethod + def get_module_configs(): + """ + The dot access dict object holds configurations block in command.json which maps service configurations + :return: + """ + return Script.module_configs + + @staticmethod def get_password(user): return Script.passwords[user] diff --git a/ambari-common/src/test/python/TestExecutionCommand.py b/ambari-common/src/test/python/TestExecutionCommand.py new file mode 100644 index 0000000..79da47f --- /dev/null +++ b/ambari-common/src/test/python/TestExecutionCommand.py @@ -0,0 +1,60 @@ +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from unittest import TestCase +from resource_management.core.logger import Logger +from resource_management.libraries.execution_command import execution_command + +import sys +import json + +command_data_file = "command.json" + +class TestExecutionCommand(TestCase): + + def setUp(self): + Logger.initialize_logger() + try: + with open(command_data_file) as f: + self.__execution_command = execution_command.ExecutionCommand(json.load(f)) + except IOError: + Logger.error("Can not read json file with command parameters: ") + sys.exit(1) + + def test_get_module_name(self): + module_name = self.__execution_command.get_module_name() + self.assertEquals(module_name, "ZOOKEEPER") + + def test_get_module_configs(self): + module_configs = self.__execution_command.get_module_configs() + self.assertNotEquals(module_configs, None) + zookeeper_client_port = module_configs.get_property_value("zookeeper", "zoo.cfg", "clientPort") + self.assertEquals(int(zookeeper_client_port), 2181) + zookeeper_client_port_fake = module_configs.get_property_value("zookeeper", "zoo.cfg", "clientPort1") + self.assertEquals(zookeeper_client_port_fake, None) + zookeeper_client_port_default_value = module_configs.get_property_value("zookeeper", "zoo.cfg", "clientPort1", 1111) + self.assertEquals(int(zookeeper_client_port_default_value), 1111) + zookeeper_empty_value = module_configs.get_property_value("zookeeper", "zoo_fake", "", {}) + self.assertEquals(zookeeper_empty_value, {}) + zookeeper_log_max_backup_size = module_configs.get_property_value('zookeeper', 'zookeeper-log4j', + 'zookeeper_log_max_backup_size', 10) + self.assertEquals(zookeeper_log_max_backup_size, 10) + + def test_get_stack_name(self): + stack_name = self.__execution_command.get_stack_name() + self.assertEquals(stack_name, "HDPCORE") \ No newline at end of file diff --git a/ambari-common/src/test/python/command.json b/ambari-common/src/test/python/command.json new file mode 100644 index 0000000..220a85c --- /dev/null +++ b/ambari-common/src/test/python/command.json @@ -0,0 +1,93 @@ +{ + "serviceName": "ZOOKEEPER", + "clusterSettings": { + "security_enabled": "false", + "smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab", + "managed_hdfs_resource_property_names": "", + "smokeuser": "ambari-qa", + "sysprep_skip_copy_fast_jar_hdfs": "false" + }, + "clusterName": "c1", + "stackSettings": { + "stack_name": "HDP", + "stack_root": "{\"HDP\":\"/usr/hdp\"}" + }, + "configurations": { + "ranger-hdfs-audit": { + "xasecure.audit.destination.solr.zookeepers": "NONE", + "xasecure.audit.destination.solr.urls": "", + "xasecure.audit.destination.solr": "false", + "xasecure.audit.destination.hdfs.batch.filespool.dir": "/var/log/hadoop/hdfs/audit/hdfs/spool", + "xasecure.audit.is.enabled": "true" + }, + "ssl-client": { + "ssl.client.truststore.reload.interval": "10000", + "ssl.client.keystore.password": "bigdata", + "ssl.client.truststore.type": "jks", + "ssl.client.keystore.location": "/etc/security/clientKeys/keystore.jks", + "ssl.client.keystore.type": "jks" + }, + "ssl-server": { + "ssl.server.keystore.location": "/etc/security/serverKeys/keystore.jks", + "ssl.server.keystore.type": "jks", + "ssl.server.truststore.reload.interval": "10000" + }, + "hdfs-site": { + "dfs.namenode.checkpoint.period": "21600", + "dfs.namenode.avoid.write.stale.datanode": "true", + "dfs.namenode.startup.delay.block.deletion.sec": "3600", + "dfs.namenode.checkpoint.edits.dir": "${dfs.namenode.checkpoint.dir}", + "dfs.namenode.rpc-address": "c7301.ambari.apache.org:8020", + "dfs.journalnode.https-address": "0.0.0.0:8481", + "dfs.journalnode.http-address": "0.0.0.0:8480", + "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket" + }, + "hdfs-logsearch-conf": {}, + "zoo.cfg": { + "clientPort": "2181", + "autopurge.purgeInterval": "24", + "syncLimit": "5", + "dataDir": "/hadoop/zookeeper", + "initLimit": "10", + "tickTime": "3000", + "autopurge.snapRetainCount": "30" + }, + "ranger-hdfs-security": { + "ranger.plugin.hdfs.policy.source.impl": "org.apache.ranger.admin.client.RangerAdminRESTClient", + "ranger.plugin.hdfs.service.name": "{{repo_name}}", + "ranger.plugin.hdfs.policy.cache.dir": "/etc/ranger/{{repo_name}}/policycache", + "ranger.plugin.hdfs.policy.pollIntervalMs": "30000", + "ranger.plugin.hdfs.policy.rest.url": "{{policymgr_mgr_url}}", + "ranger.plugin.hdfs.policy.rest.ssl.config.file": "/etc/hadoop/conf/ranger-policymgr-ssl.xml", + "xasecure.add-hadoop-authorization": "true" + }, + "core-site": { + "hadoop.custom-extensions.root": "/hdp/ext/{{major_stack_version}}/hadoop", + "io.serializations": "org.apache.hadoop.io.serializer.WritableSerialization", + "fs.trash.interval": "360", + "hadoop.proxyuser.root.groups": "*", + "ipc.client.connection.maxidletime": "30000", + "net.topology.script.file.name": "/etc/hadoop/conf/topology_script.py", + "ipc.server.tcpnodelay": "true", + "fs.defaultFS": "hdfs://c7301.ambari.apache.org:8020" + }, + "hadoop-policy": { + "security.job.client.protocol.acl": "*", + "security.job.task.protocol.acl": "*" + }, + "zookeeper-env": { + "zk_log_dir": "/var/log/zookeeper", + "content": "\nexport JAVA_HOME={{java64_home}}\nexport ZOOKEEPER_HOME={{zk_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login. [...] + "zk_server_heapsize": "1024m", + "zk_pid_dir": "/var/run/zookeeper", + "zk_user": "zookeeper" + } + }, + "hostLevelParams": { + "stack_name": "HDPCORE", + "group_list": "[\"hdfs\",\"users\"]", + "host_sys_prepped": "false", + "jdk_name": "jdk-8u112-linux-x64.tar.gz", + "stack_version": "1.0.0-b141" + } +} \ No newline at end of file -- To stop receiving notification emails like this one, please contact jlun...@apache.org.