This is an automated email from the ASF dual-hosted git repository.
jluniya pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by
this push:
new a33db89 Ambari-23136: Create an interface to read command.json
properties via API instead of direct access (#709)
a33db89 is described below
commit a33db89f0a311b2bcf5c046b6209ccd7dcae3a9d
Author: sduan <sd...@hortonworks.com>
AuthorDate: Fri Mar 23 08:17:22 2018 -0700
Ambari-23136: Create an interface to read command.json properties via API
instead of direct access (#709)
* AMBARI-23136: Create an interface to read command.json properties via API
instead of direct access
* AMBARI-23136: Create an interface to read command.json properties via API
instead of direct access
---
.../resource_management/libraries/__init__.py | 1 +
.../libraries/{ => execution_command}/__init__.py | 6 --
.../execution_command/execution_command.py | 85 ++++++++++++++++++++
.../module_configs.py} | 27 +++++--
.../resource_management/libraries/script/script.py | 22 +++++
.../src/test/python/TestExecutionCommand.py | 60 ++++++++++++++
ambari-common/src/test/python/command.json | 93 ++++++++++++++++++++++
7 files changed, 282 insertions(+), 12 deletions(-)
diff --git
a/ambari-common/src/main/python/resource_management/libraries/__init__.py
b/ambari-common/src/main/python/resource_management/libraries/__init__.py
index b1d3a36..b36d4a2 100644
--- a/ambari-common/src/main/python/resource_management/libraries/__init__.py
+++ b/ambari-common/src/main/python/resource_management/libraries/__init__.py
@@ -20,6 +20,7 @@ Ambari Agent
"""
+from resource_management.libraries.execution_command import *
from resource_management.libraries.functions import *
from resource_management.libraries.resources import *
from resource_management.libraries.providers import *
diff --git
a/ambari-common/src/main/python/resource_management/libraries/__init__.py
b/ambari-common/src/main/python/resource_management/libraries/execution_command/__init__.py
similarity index 77%
copy from
ambari-common/src/main/python/resource_management/libraries/__init__.py
copy to
ambari-common/src/main/python/resource_management/libraries/execution_command/__init__.py
index b1d3a36..d05d43b 100644
--- a/ambari-common/src/main/python/resource_management/libraries/__init__.py
+++
b/ambari-common/src/main/python/resource_management/libraries/execution_command/__init__.py
@@ -16,11 +16,5 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
or implied.
See the License for the specific language governing permissions and
limitations under the License.
-Ambari Agent
-
"""
-from resource_management.libraries.functions import *
-from resource_management.libraries.resources import *
-from resource_management.libraries.providers import *
-from resource_management.libraries.script import *
diff --git
a/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py
b/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py
new file mode 100644
index 0000000..0e3f8e4
--- /dev/null
+++
b/ambari-common/src/main/python/resource_management/libraries/execution_command/execution_command.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+__all__ = ["ExecutionCommand"]
+
+import ambari_simplejson
+
+from resource_management.libraries.execution_command import module_configs
+
+
+class ExecutionCommand(object):
+ """
+ The class maps to a command.json. All command related info should be
retrieved from this class object
+ """
+
+ def __init__(self, command):
+ """
+ __execution_command is an internal dot access dict object maps to
command.json
+ :param command: json string or a python dict object
+ """
+ self.__execution_command = command
+ self.__module_configs =
module_configs.ModuleConfigs(self.__get_value("configurations"))
+
+ def __get_value(self, key, default_value=None):
+ """
+ A private method to query value with the full path of key, if key does not
exist, return default_value
+ :param key:
+ :param default_value:
+ :return:
+ """
+ sub_keys = key.split('/')
+ value = self.__execution_command
+ try:
+ for sub_key in sub_keys:
+ value = value[sub_key]
+ return value
+ except:
+ return default_value
+
+ def get_module_name(self):
+ return self.__get_value("serviceName")
+
+ def get_stack_name(self):
+ return self.__get_value("hostLevelParams/stack_name")
+
+ def get_stack_version(self):
+ return self.__get_value("hostLevelParams/stack_version")
+
+ def get_new_stack_version_for_upgrade(self):
+ """
+ New Cluster Stack Version that is defined during the RESTART of a Rolling
Upgrade
+ :return:
+ """
+ return self.__get_value("commandParams/version")
+
+ def get_host_name(self):
+ return self.__get_value("hostname")
+
+ def get_java_home(self):
+ return self.__get_value("hostLevelParams/java_home")
+
+ def get_java_version(self):
+ java_version = self.__get_value("hostLevelParams/java_version")
+ return int(java_version) if java_version else None
+
+ def get_module_configs(self):
+ return self.__module_configs
+
diff --git
a/ambari-common/src/main/python/resource_management/libraries/__init__.py
b/ambari-common/src/main/python/resource_management/libraries/execution_command/module_configs.py
similarity index 55%
copy from
ambari-common/src/main/python/resource_management/libraries/__init__.py
copy to
ambari-common/src/main/python/resource_management/libraries/execution_command/module_configs.py
index b1d3a36..967449f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/__init__.py
+++
b/ambari-common/src/main/python/resource_management/libraries/execution_command/module_configs.py
@@ -16,11 +16,26 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-Ambari Agent
-
"""
-from resource_management.libraries.functions import *
-from resource_management.libraries.resources import *
-from resource_management.libraries.providers import *
-from resource_management.libraries.script import *
+__all__ = ["ModuleConfigs"]
+
+
+class ModuleConfigs(object):
+ """
+ This class maps to "/configurations" block in command.json which includes
configuration information of a service
+ """
+
+ def __init__(self, config):
+ self.__module_configs = config
+
+ def get_properties(self, module_name, config_type, property_names):
+ return map(lambda property_name: self.get_property_value(module_name,
config_type, property_name), property_names)
+
+ def get_property_value(self, module_name, config_type, property_name,
default=None):
+ try:
+ return self.__module_configs[config_type][property_name]
+ except:
+ return default
+
+
diff --git
a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index cd528c1..878c83a 100644
---
a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -64,6 +64,8 @@ from resource_management.core.providers import get_provider
from resource_management.libraries.functions.fcntl_based_process_lock import
FcntlBasedProcessLock
from resource_management.libraries.functions.config_helper import
get_mpack_name, get_mpack_version, \
get_mpack_instance_name, get_module_name, get_component_type,
get_component_instance_name
+from resource_management.libraries.execution_command.execution_command import
ExecutionCommand
+from resource_management.libraries.execution_command.module_configs import
ModuleConfigs
import ambari_simplejson as json # simplejson is much faster comparing to
Python 2.6 json module and has the same functions set.
@@ -146,6 +148,8 @@ class Script(object):
4 path to file with structured command output (file will be created)
"""
config = None
+ execution_command = None
+ module_configs = None
stack_version_from_distro_select = None
structuredOut = {}
command_data_file = ""
@@ -339,6 +343,8 @@ class Script(object):
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
+ Script.execution_command = ExecutionCommand(Script.config)
+ Script.module_configs = Script.execution_command.get_module_configs()
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
@@ -598,6 +604,22 @@ class Script(object):
return Script.config
@staticmethod
+ def get_execution_command():
+ """
+ The dot access dict object holds command.json
+ :return:
+ """
+ return Script.execution_command
+
+ @staticmethod
+ def get_module_configs():
+ """
+ The dot access dict object holds configurations block in command.json
which maps service configurations
+ :return:
+ """
+ return Script.module_configs
+
+ @staticmethod
def get_password(user):
return Script.passwords[user]
diff --git a/ambari-common/src/test/python/TestExecutionCommand.py
b/ambari-common/src/test/python/TestExecutionCommand.py
new file mode 100644
index 0000000..79da47f
--- /dev/null
+++ b/ambari-common/src/test/python/TestExecutionCommand.py
@@ -0,0 +1,60 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from resource_management.core.logger import Logger
+from resource_management.libraries.execution_command import execution_command
+
+import sys
+import json
+
+command_data_file = "command.json"
+
+class TestExecutionCommand(TestCase):
+
+ def setUp(self):
+ Logger.initialize_logger()
+ try:
+ with open(command_data_file) as f:
+ self.__execution_command =
execution_command.ExecutionCommand(json.load(f))
+ except IOError:
+ Logger.error("Can not read json file with command parameters: ")
+ sys.exit(1)
+
+ def test_get_module_name(self):
+ module_name = self.__execution_command.get_module_name()
+ self.assertEquals(module_name, "ZOOKEEPER")
+
+ def test_get_module_configs(self):
+ module_configs = self.__execution_command.get_module_configs()
+ self.assertNotEquals(module_configs, None)
+ zookeeper_client_port = module_configs.get_property_value("zookeeper",
"zoo.cfg", "clientPort")
+ self.assertEquals(int(zookeeper_client_port), 2181)
+ zookeeper_client_port_fake =
module_configs.get_property_value("zookeeper", "zoo.cfg", "clientPort1")
+ self.assertEquals(zookeeper_client_port_fake, None)
+ zookeeper_client_port_default_value =
module_configs.get_property_value("zookeeper", "zoo.cfg", "clientPort1", 1111)
+ self.assertEquals(int(zookeeper_client_port_default_value), 1111)
+ zookeeper_empty_value = module_configs.get_property_value("zookeeper",
"zoo_fake", "", {})
+ self.assertEquals(zookeeper_empty_value, {})
+ zookeeper_log_max_backup_size =
module_configs.get_property_value('zookeeper', 'zookeeper-log4j',
+
'zookeeper_log_max_backup_size', 10)
+ self.assertEquals(zookeeper_log_max_backup_size, 10)
+
+ def test_get_stack_name(self):
+ stack_name = self.__execution_command.get_stack_name()
+ self.assertEquals(stack_name, "HDPCORE")
\ No newline at end of file
diff --git a/ambari-common/src/test/python/command.json
b/ambari-common/src/test/python/command.json
new file mode 100644
index 0000000..220a85c
--- /dev/null
+++ b/ambari-common/src/test/python/command.json
@@ -0,0 +1,93 @@
+{
+ "serviceName": "ZOOKEEPER",
+ "clusterSettings": {
+ "security_enabled": "false",
+ "smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab",
+ "managed_hdfs_resource_property_names": "",
+ "smokeuser": "ambari-qa",
+ "sysprep_skip_copy_fast_jar_hdfs": "false"
+ },
+ "clusterName": "c1",
+ "stackSettings": {
+ "stack_name": "HDP",
+ "stack_root": "{\"HDP\":\"/usr/hdp\"}"
+ },
+ "configurations": {
+ "ranger-hdfs-audit": {
+ "xasecure.audit.destination.solr.zookeepers": "NONE",
+ "xasecure.audit.destination.solr.urls": "",
+ "xasecure.audit.destination.solr": "false",
+ "xasecure.audit.destination.hdfs.batch.filespool.dir":
"/var/log/hadoop/hdfs/audit/hdfs/spool",
+ "xasecure.audit.is.enabled": "true"
+ },
+ "ssl-client": {
+ "ssl.client.truststore.reload.interval": "10000",
+ "ssl.client.keystore.password": "bigdata",
+ "ssl.client.truststore.type": "jks",
+ "ssl.client.keystore.location": "/etc/security/clientKeys/keystore.jks",
+ "ssl.client.keystore.type": "jks"
+ },
+ "ssl-server": {
+ "ssl.server.keystore.location": "/etc/security/serverKeys/keystore.jks",
+ "ssl.server.keystore.type": "jks",
+ "ssl.server.truststore.reload.interval": "10000"
+ },
+ "hdfs-site": {
+ "dfs.namenode.checkpoint.period": "21600",
+ "dfs.namenode.avoid.write.stale.datanode": "true",
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600",
+ "dfs.namenode.checkpoint.edits.dir": "${dfs.namenode.checkpoint.dir}",
+ "dfs.namenode.rpc-address": "c7301.ambari.apache.org:8020",
+ "dfs.journalnode.https-address": "0.0.0.0:8481",
+ "dfs.journalnode.http-address": "0.0.0.0:8480",
+ "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket"
+ },
+ "hdfs-logsearch-conf": {},
+ "zoo.cfg": {
+ "clientPort": "2181",
+ "autopurge.purgeInterval": "24",
+ "syncLimit": "5",
+ "dataDir": "/hadoop/zookeeper",
+ "initLimit": "10",
+ "tickTime": "3000",
+ "autopurge.snapRetainCount": "30"
+ },
+ "ranger-hdfs-security": {
+ "ranger.plugin.hdfs.policy.source.impl":
"org.apache.ranger.admin.client.RangerAdminRESTClient",
+ "ranger.plugin.hdfs.service.name": "{{repo_name}}",
+ "ranger.plugin.hdfs.policy.cache.dir":
"/etc/ranger/{{repo_name}}/policycache",
+ "ranger.plugin.hdfs.policy.pollIntervalMs": "30000",
+ "ranger.plugin.hdfs.policy.rest.url": "{{policymgr_mgr_url}}",
+ "ranger.plugin.hdfs.policy.rest.ssl.config.file":
"/etc/hadoop/conf/ranger-policymgr-ssl.xml",
+ "xasecure.add-hadoop-authorization": "true"
+ },
+ "core-site": {
+ "hadoop.custom-extensions.root":
"/hdp/ext/{{major_stack_version}}/hadoop",
+ "io.serializations":
"org.apache.hadoop.io.serializer.WritableSerialization",
+ "fs.trash.interval": "360",
+ "hadoop.proxyuser.root.groups": "*",
+ "ipc.client.connection.maxidletime": "30000",
+ "net.topology.script.file.name": "/etc/hadoop/conf/topology_script.py",
+ "ipc.server.tcpnodelay": "true",
+ "fs.defaultFS": "hdfs://c7301.ambari.apache.org:8020"
+ },
+ "hadoop-policy": {
+ "security.job.client.protocol.acl": "*",
+ "security.job.task.protocol.acl": "*"
+ },
+ "zookeeper-env": {
+ "zk_log_dir": "/var/log/zookeeper",
+ "content": "\nexport JAVA_HOME={{java64_home}}\nexport
ZOOKEEPER_HOME={{zk_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport
ZOOPIDFILE={{zk_pid_file}}\nexport
SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport
CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled
%}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS
-Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport
CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login. [...]
+ "zk_server_heapsize": "1024m",
+ "zk_pid_dir": "/var/run/zookeeper",
+ "zk_user": "zookeeper"
+ }
+ },
+ "hostLevelParams": {
+ "stack_name": "HDPCORE",
+ "group_list": "[\"hdfs\",\"users\"]",
+ "host_sys_prepped": "false",
+ "jdk_name": "jdk-8u112-linux-x64.tar.gz",
+ "stack_version": "1.0.0-b141"
+ }
+}
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
jlun...@apache.org.
