Repository: ambari Updated Branches: refs/heads/trunk dd888838d -> 55bbcae46
http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py new file mode 100644 index 0000000..bee2f36 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +import tarfile +import tempfile + +__all__ = ["Script"] + +import os +import sys +import json +import logging +from contextlib import closing + + +from resource_management.libraries.resources import XmlConfig +from resource_management.core.resources import File, Directory +from resource_management.core.source import InlineTemplate + +from resource_management.core.environment import Environment +from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus, ComponentIsNotRunning +from resource_management.core.resources.packaging import Package +from resource_management.libraries.script.config_dictionary import ConfigDictionary + + +USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEVEL> <TMP_DIR> + +<COMMAND> command type (INSTALL/CONFIGURE/START/STOP/SERVICE_CHECK...) +<JSON_CONFIG> path to command json file. Ex: /var/lib/ambari-agent/data/command-2.json +<BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/stacks/HDP/2.0.6/services/HDFS +<STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt +<LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO +<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/data/tmp +""" + +class Script(object): + """ + Executes a command for custom service. stdout and stderr are written to + tmpoutfile and to tmperrfile respectively. + Script instances share configuration as a class parameter and therefore + different Script instances can not be used from different threads at + the same time within a single python process + + Accepted command line arguments mapping: + 1 command type (START/STOP/...) + 2 path to command json file + 3 path to service metadata dir (Directory "package" inside service directory) + 4 path to file with structured command output (file will be created) + """ + structuredOut = {} + + def put_structured_out(self, sout): + Script.structuredOut.update(sout) + try: + with open(self.stroutfile, 'w') as fp: + json.dump(Script.structuredOut, fp) + except IOError: + Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile}) + + def execute(self): + """ + Sets up logging; + Parses command parameters and executes method relevant to command type + """ + # set up logging (two separate loggers for stderr and stdout with different loglevels) + logger = logging.getLogger('resource_management') + logger.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(message)s') + chout = logging.StreamHandler(sys.stdout) + chout.setLevel(logging.INFO) + chout.setFormatter(formatter) + cherr = logging.StreamHandler(sys.stderr) + cherr.setLevel(logging.ERROR) + cherr.setFormatter(formatter) + logger.addHandler(cherr) + logger.addHandler(chout) + + # parse arguments + if len(sys.argv) < 7: + logger.error("Script expects at least 6 arguments") + print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout + sys.exit(1) + + command_name = str.lower(sys.argv[1]) + command_data_file = sys.argv[2] + basedir = sys.argv[3] + self.stroutfile = sys.argv[4] + logging_level = sys.argv[5] + Script.tmp_dir = sys.argv[6] + + logging_level_str = logging._levelNames[logging_level] + chout.setLevel(logging_level_str) + logger.setLevel(logging_level_str) + + try: + with open(command_data_file, "r") as f: + pass + Script.config = ConfigDictionary(json.load(f)) + except IOError: + logger.exception("Can not read json file with command parameters: ") + sys.exit(1) + # Run class method depending on a command type + try: + method = self.choose_method_to_execute(command_name) + with Environment(basedir) as env: + method(env) + except ClientComponentHasNoStatus or ComponentIsNotRunning: + # Support of component status checks. + # Non-zero exit code is interpreted as an INSTALLED status of a component + sys.exit(1) + except Fail: + logger.exception("Error while executing command '{0}':".format(command_name)) + sys.exit(1) + + + def choose_method_to_execute(self, command_name): + """ + Returns a callable object that should be executed for a given command. + """ + self_methods = dir(self) + if not command_name in self_methods: + raise Fail("Script '{0}' has no method '{1}'".format(sys.argv[0], command_name)) + method = getattr(self, command_name) + return method + + + @staticmethod + def get_config(): + """ + HACK. Uses static field to store configuration. This is a workaround for + "circular dependency" issue when importing params.py file and passing to + it a configuration instance. + """ + return Script.config + + + @staticmethod + def get_tmp_dir(): + """ + HACK. Uses static field to avoid "circular dependency" issue when + importing params.py. + """ + return Script.tmp_dir + + + def install(self, env): + """ + Default implementation of install command is to install all packages + from a list, received from the server. + Feel free to override install() method with your implementation. It + usually makes sense to call install_packages() manually in this case + """ + self.install_packages(env) + + + def install_packages(self, env, exclude_packages=[]): + """ + List of packages that are required< by service is received from the server + as a command parameter. The method installs all packages + from this list + """ + config = self.get_config() + + try: + package_list_str = config['hostLevelParams']['package_list'] + if isinstance(package_list_str,basestring) and len(package_list_str) > 0: + package_list = json.loads(package_list_str) + for package in package_list: + if not package['name'] in exclude_packages: + name = package['name'] + Package(name) + except KeyError: + pass # No reason to worry + + #RepoInstaller.remove_repos(config) + + + + def fail_with_error(self, message): + """ + Prints error message and exits with non-zero exit code + """ + print("Error: " + message) + sys.stderr.write("Error: " + message) + sys.exit(1) + + def start(self, env): + """ + To be overridden by subclasses + """ + self.fail_with_error('start method isn\'t implemented') + + def stop(self, env): + """ + To be overridden by subclasses + """ + self.fail_with_error('stop method isn\'t implemented') + + def restart(self, env): + """ + Default implementation of restart command is to call stop and start methods + Feel free to override restart() method with your implementation. + For client components we call install + """ + config = self.get_config() + componentCategory = None + try : + componentCategory = config['roleParams']['component_category'] + except KeyError: + pass + + if componentCategory and componentCategory.strip().lower() == 'CLIENT'.lower(): + self.install(env) + else: + self.stop(env) + self.start(env) + + def configure(self, env): + """ + To be overridden by subclasses + """ + self.fail_with_error('configure method isn\'t implemented') + + def generate_configs(self, env): + """ + Generates config files and stores them as an archive in tmp_dir + based on xml_configs_list and env_configs_list from commandParams + """ + import params + config = self.get_config() + xml_configs_list = json.loads(config['commandParams']['xml_configs_list']) + env_configs_list = json.loads(config['commandParams']['env_configs_list']) + conf_tmp_dir = tempfile.mkdtemp() + output_filename = os.path.join(self.get_tmp_dir(),"client-configs.tar.gz") + + Directory(self.get_tmp_dir(), recursive=True) + for file_dict in xml_configs_list: + for filename, dict in file_dict.iteritems(): + XmlConfig(filename, + conf_dir=conf_tmp_dir, + configurations=params.config['configurations'][dict], + configuration_attributes=params.config['configuration_attributes'][dict], + ) + for file_dict in env_configs_list: + for filename,dict in file_dict.iteritems(): + File(os.path.join(conf_tmp_dir, filename), + content=InlineTemplate(params.config['configurations'][dict]['content']) + ) + with closing(tarfile.open(output_filename, "w:gz")) as tar: + tar.add(conf_tmp_dir, arcname=os.path.basename(".")) + Directory(conf_tmp_dir, action="delete") http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-server/conf/unix/install-helper.sh ---------------------------------------------------------------------- diff --git a/ambari-server/conf/unix/install-helper.sh b/ambari-server/conf/unix/install-helper.sh index 740c2cc..dafe987 100644 --- a/ambari-server/conf/unix/install-helper.sh +++ b/ambari-server/conf/unix/install-helper.sh @@ -18,9 +18,11 @@ ################################################################## COMMON_DIR="/usr/lib/python2.6/site-packages/ambari_commons" +RESOURCE_MANAGEMENT_DIR="/usr/lib/python2.6/site-packages/resource_management" OLD_COMMON_DIR="/usr/lib/python2.6/site-packages/common_functions" INSTALL_HELPER_AGENT="/var/lib/ambari-agent/install-helper.sh" COMMON_DIR_SERVER="/usr/lib/ambari-server/lib/ambari_commons" +RESOURCE_MANAGEMENT_DIR_SERVER="/usr/lib/ambari-server/lib/resource_management" PYTHON_WRAPER_TARGET="/usr/bin/ambari-python-wrap" PYTHON_WRAPER_SOURCE="/var/lib/ambari-server/ambari-python-wrap" @@ -31,6 +33,10 @@ do_install(){ if [ ! -d "$COMMON_DIR" ]; then ln -s "$COMMON_DIR_SERVER" "$COMMON_DIR" fi + # setting resource_management shared resource + if [ ! -d "$RESOURCE_MANAGEMENT_DIR" ]; then + ln -s "$RESOURCE_MANAGEMENT_DIR_SERVER" "$RESOURCE_MANAGEMENT_DIR" + fi # setting python-wrapper script if [ ! -f "$PYTHON_WRAPER_TARGET" ]; then ln -s "$PYTHON_WRAPER_SOURCE" "$PYTHON_WRAPER_TARGET" @@ -40,6 +46,7 @@ do_install(){ do_remove(){ rm -rf "$COMMON_DIR" + rm -rf "$RESOURCE_MANAGEMENT_DIR" if [ -f "$PYTHON_WRAPER_TARGET" ]; then rm -f "$PYTHON_WRAPER_TARGET" http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index a6d03ab..1c5f5d5 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -32,6 +32,7 @@ <hdpUrlForCentos6>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.1.0</hdpUrlForCentos6> <hdpLatestUrl>http://public-repo-1.hortonworks.com/HDP/hdp_urlinfo.json</hdpLatestUrl> <ambari_commons.install.dir>/usr/lib/ambari-server/lib/ambari_commons</ambari_commons.install.dir> + <resource_management.install.dir>/usr/lib/ambari-server/lib/resource_management</resource_management.install.dir> <ambari-web-dir>${basedir}/../ambari-web/public</ambari-web-dir> <ambari-admin-dir>${basedir}/../ambari-admin</ambari-admin-dir> </properties> @@ -264,6 +265,16 @@ </source> </sources> </mapping> + <mapping> + <directory>${resource_management.install.dir}</directory> + <sources> + <source> + <location> + ${project.basedir}/../ambari-common/src/main/python/resource_management + </location> + </source> + </sources> + </mapping> <mapping> <directory>/usr/sbin</directory> <filemode>755</filemode> @@ -864,6 +875,19 @@ <group>root</group> </mapper> </data> + <data> + <src> + ${project.basedir}/../ambari-common/src/main/python/resource_management + </src> + <type>directory</type> + <mapper> + <type>perm</type> + <prefix>${resource_management.install.dir}</prefix> + <filemode>755</filemode> + <user>root</user> + <group>root</group> + </mapper> + </data> </dataSet> </configuration> </plugin> @@ -897,7 +921,7 @@ <argument>${custom.tests}</argument> </arguments> <environmentVariables> - <PYTHONPATH>${project.basedir}/../ambari-agent/src/main/python:${project.basedir}/../ambari-common/src/main/python/jinja2:${project.basedir}/../ambari-common/src/main/python:${project.basedir}/../ambari-common/src/main/python/ambari_commons:${project.basedir}/../ambari-common/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/ambari-server-state:${project.basedir}/src/test/python:$PYTHONPATH</PYTHONPATH> + <PYTHONPATH>${project.basedir}/../ambari-agent/src/main/python:${project.basedir}/../ambari-common/src/main/python/jinja2:${project.basedir}/../ambari-common/src/main/python:${project.basedir}/../ambari-common/src/main/python/ambari_commons:${project.basedir}/../ambari-common/src/main/python/resource_management:${project.basedir}/../ambari-common/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/ambari-server-state:${project.basedir}/src/test/python:$PYTHONPATH</PYTHONPATH> </environmentVariables> <skip>${skipTests}</skip> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py new file mode 100644 index 0000000..2f2ca5a --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +from mock.mock import MagicMock, patch +import tempfile +import tarfile +import contextlib +from resource_management import * +from stacks.utils.RMFTestCase import * + + +@patch.object(tarfile,"open", new = MagicMock()) +@patch.object(tempfile,"mkdtemp", new = MagicMock(return_value='/tmp/123')) +@patch.object(contextlib,"closing", new = MagicMock()) +@patch("os.path.exists", new = MagicMock(return_value=True)) +class TestDatanode(RMFTestCase): + + def test_generate_configs_default(self): + self.executeScript("2.0.6/services/HDFS/package/scripts/hdfs_client.py", + classname = "HdfsClient", + command = "generate_configs", + config_file="default.json" + ) + self.assertResourceCalled('Directory', '/tmp', + recursive = True, + ) + self.assertResourceCalled('XmlConfig', 'hdfs-site.xml', + conf_dir = '/tmp/123', + configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site'], + configurations = self.getConfig()['configurations']['hdfs-site'], + ) + self.assertResourceCalled('File', '/tmp/123/hadoop-env.sh', + content = InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']), + ) + self.assertResourceCalled('Directory', '/tmp/123', + action = ['delete'], + ) + self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index cc69ef8..beeb3e7 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -26,7 +26,10 @@ "script_type": "PYTHON", "script": "scripts/service_check.py", "excluded_hosts": "host1,host2", - "mark_draining_only" : "false" + "mark_draining_only" : "false", + "xml_configs_list":"[{\"hdfs-site.xml\":\"hdfs-site\"}]", + "env_configs_list":"[{\"hadoop-env.sh\":\"hadoop-env\"}]" + }, "taskId": 152, "public_hostname": "c6401.ambari.apache.org",
