Repository: ambari Updated Branches: refs/heads/branch-windows-dev 2868c2051 -> 9cff20bf1
AMBARI-8130. Provide ability to download client configurations in Windows (Florian Barca via alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9cff20bf Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9cff20bf Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9cff20bf Branch: refs/heads/branch-windows-dev Commit: 9cff20bf1634a8b242a8b150c41ae5a93888adcb Parents: 2868c20 Author: Alejandro Fernandez <afernan...@hortonworks.com> Authored: Thu Nov 6 16:39:31 2014 -0800 Committer: Alejandro Fernandez <afernan...@hortonworks.com> Committed: Thu Nov 6 16:39:31 2014 -0800 ---------------------------------------------------------------------- .../libraries/functions/tar_archive.py | 30 ++++++++++ .../libraries/functions/zip_archive.py | 40 +++++++++++++ .../libraries/providers/xml_config.py | 6 +- .../libraries/script/script.py | 59 ++++++++++---------- ambari-server/conf/unix/ambari.properties | 1 + ambari-server/conf/windows/ambari.properties | 1 + .../src/main/assemblies/server-windows.xml | 8 +++ .../server/api/services/ComponentService.java | 6 +- .../server/configuration/Configuration.java | 19 +++++++ .../internal/ClientConfigResourceProvider.java | 7 ++- 10 files changed, 141 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py new file mode 100644 index 0000000..efbf933 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import os +import tarfile +from contextlib import closing + +def archive_dir(output_filename, input_dir): + with closing(tarfile.open(output_filename, "w:gz")) as tar: + try: + tar.add(input_dir, arcname=os.path.basename(".")) + finally: + tar.close() http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py new file mode 100644 index 0000000..cab3627 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import os +import zipfile + +def _zip_dir(zip, root): + for dirname, dirnames, filenames in os.walk(root): + for filename in filenames: + if len(dirname) > len(root): + rel_path = os.path.relpath(dirname, root) + arch_name = rel_path + os.sep + filename + else: + arch_name = filename + zip.write(os.path.join(dirname, filename), arch_name) + + +def archive_dir(output_filename, input_dir): + zipf = zipfile.ZipFile(output_filename, 'w') + try: + _zip_dir(zipf, input_dir) + finally: + zipf.close() http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py index 34b43e3..a7b2c21 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py @@ -49,11 +49,11 @@ class XmlConfigProvider(Provider): </configuration>''', extra_imports=[time], configurations_dict=self.resource.configurations, configuration_attrs=self.resource.configuration_attributes) - - Logger.info(format("Generating config: {xml_config_provider_config_dir}/{filename}")) + xml_config_dest_file_path = os.path.join(xml_config_provider_config_dir, filename) + Logger.info("Generating config: {}".format(xml_config_dest_file_path)) with Environment.get_instance_copy() as env: - File (format("{xml_config_provider_config_dir}/{filename}"), + File (xml_config_dest_file_path, content = config_content, owner = self.resource.owner, group = self.resource.group, http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 059dd57..d58502a 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -17,7 +17,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' -import tarfile import tempfile __all__ = ["Script"] @@ -27,7 +26,6 @@ import sys import json import logging import platform -from contextlib import closing from resource_management.libraries.resources import XmlConfig from resource_management.core.resources import File, Directory @@ -41,6 +39,9 @@ IS_WINDOWS = platform.system() == "Windows" if IS_WINDOWS: from resource_management.libraries.functions.install_hdp_msi import install_windows_msi from resource_management.libraries.functions.reload_windows_env import reload_windows_env + from resource_management.libraries.functions.zip_archive import archive_dir +else: + from resource_management.libraries.functions.tar_archive import archive_dir USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEVEL> <TMP_DIR> @@ -280,43 +281,45 @@ class Script(object): self.fail_with_error('configure method isn\'t implemented') def generate_configs_get_template_file_content(self, filename, dicts): - import params + config = self.get_config() content = '' for dict in dicts.split(','): - if dict.strip() in params.config['configurations']: - content += params.config['configurations'][dict.strip()]['content'] + if dict.strip() in config['configurations']: + content += config['configurations'][dict.strip()]['content'] return content def generate_configs_get_xml_file_content(self, filename, dict): - import params - return {'configurations':params.config['configurations'][dict], - 'configuration_attributes':params.config['configuration_attributes'][dict]} + config = self.get_config() + return {'configurations':config['configurations'][dict], + 'configuration_attributes':config['configuration_attributes'][dict]} def generate_configs(self, env): """ Generates config files and stores them as an archive in tmp_dir based on xml_configs_list and env_configs_list from commandParams """ - import params - env.set_params(params) - xml_configs_list = params.config['commandParams']['xml_configs_list'] - env_configs_list = params.config['commandParams']['env_configs_list'] - conf_tmp_dir = tempfile.mkdtemp() - output_filename = os.path.join(self.get_tmp_dir(),params.config['commandParams']['output_file']) + config = self.get_config() + + xml_configs_list = config['commandParams']['xml_configs_list'] + env_configs_list = config['commandParams']['env_configs_list'] Directory(self.get_tmp_dir(), recursive=True) - for file_dict in xml_configs_list: - for filename, dict in file_dict.iteritems(): - XmlConfig(filename, - conf_dir=conf_tmp_dir, - **self.generate_configs_get_xml_file_content(filename, dict) - ) - for file_dict in env_configs_list: - for filename,dicts in file_dict.iteritems(): - File(os.path.join(conf_tmp_dir, filename), - content=InlineTemplate(self.generate_configs_get_template_file_content(filename, dicts))) - with closing(tarfile.open(output_filename, "w:gz")) as tar: - tar.add(conf_tmp_dir, arcname=os.path.basename(".")) - tar.close() - Directory(conf_tmp_dir, action="delete") + + conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir()) + output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file']) + + try: + for file_dict in xml_configs_list: + for filename, dict in file_dict.iteritems(): + XmlConfig(filename, + conf_dir=conf_tmp_dir, + **self.generate_configs_get_xml_file_content(filename, dict) + ) + for file_dict in env_configs_list: + for filename,dicts in file_dict.iteritems(): + File(os.path.join(conf_tmp_dir, filename), + content=InlineTemplate(self.generate_configs_get_template_file_content(filename, dicts))) + archive_dir(output_filename, conf_tmp_dir) + finally: + Directory(conf_tmp_dir, action="delete") http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/conf/unix/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties index 531d717..771f711 100644 --- a/ambari-server/conf/unix/ambari.properties +++ b/ambari-server/conf/unix/ambari.properties @@ -33,6 +33,7 @@ bootstrap.setup_agent.script=/usr/lib/python2.6/site-packages/ambari_server/setu recommendations.dir=/var/run/ambari-server/stack-recommendations stackadvisor.script=/var/lib/ambari-server/resources/scripts/stack_advisor.py server.tmp.dir=/var/lib/ambari-server/tmp +ambari.python.wrap=ambari-python-wrap api.authenticate=true server.connection.max.idle.millis=900000 http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/conf/windows/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties index f67e7e2..fd3a7ba 100644 --- a/ambari-server/conf/windows/ambari.properties +++ b/ambari-server/conf/windows/ambari.properties @@ -52,6 +52,7 @@ recommendations.dir=\\var\\run\\ambari-server\\stack-recommendations stackadvisor.script=resources\\scripts\\stack_advisor.py server.tmp.dir=\\var\\run\\ambari-server\\tmp views.dir=resources\\views +ambari.python.wrap=python.exe # Default timeout in seconds before task is killed agent.task.timeout=600 http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/assemblies/server-windows.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/assemblies/server-windows.xml b/ambari-server/src/main/assemblies/server-windows.xml index 84029dd..e48d3d8 100644 --- a/ambari-server/src/main/assemblies/server-windows.xml +++ b/ambari-server/src/main/assemblies/server-windows.xml @@ -117,6 +117,14 @@ <directory>${basedir}/../ambari-common/src/main/python/ambari_commons</directory> <outputDirectory>ambari-server-${project.version}/sbin/ambari_commons</outputDirectory> </fileSet> + <fileSet> + <directory>${basedir}/../ambari-common/src/main/python/ambari_jinja2/ambari_jinja2</directory> + <outputDirectory>ambari-server-${project.version}/sbin/ambari_jinja2</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/../ambari-common/src/main/python/resource_management</directory> + <outputDirectory>ambari-server-${project.version}/sbin/resource_management</outputDirectory> + </fileSet> <!-- <fileSet> <directory>${project.build.directory}/web/</directory> http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java index 5510697..e7e0029 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java @@ -234,7 +234,7 @@ public class ComponentService extends BaseService { Response.ResponseBuilder rb = Response.status(Response.Status.OK); Configuration configs = new Configuration(); String tmpDir = configs.getProperty(Configuration.SERVER_TMP_DIR_KEY); - File file = new File(tmpDir+File.separator+componentName+"-configs.tar.gz"); + File file = new File(tmpDir + File.separator + componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION); InputStream resultInputStream = null; try { resultInputStream = new FileInputStream(file); @@ -242,8 +242,8 @@ public class ComponentService extends BaseService { e.printStackTrace(); } - String contentType = "application/x-ustar"; - String outputFileName = componentName + "-configs.tar.gz"; + String contentType = Configuration.DEF_ARCHIVE_CONTENT_TYPE; + String outputFileName = componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION; rb.header("Content-Disposition", "attachment; filename=\"" + outputFileName + "\""); rb.entity(resultInputStream); return rb.type(contentType).build(); http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index c1b5dcb..56bb5ab 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -71,6 +71,8 @@ public class Configuration { public static final String RECOMMENDATIONS_DIR_DEFAULT = "/var/run/ambari-server/stack-recommendations"; public static final String STACK_ADVISOR_SCRIPT = "stackadvisor.script"; public static final String STACK_ADVISOR_SCRIPT_DEFAULT = "/var/lib/ambari-server/resources/scripts/stack_advisor.py"; + public static final String AMBARI_PYTHON_WRAP_KEY = "ambari.python.wrap"; + public static final String AMBARI_PYTHON_WRAP_DEFAULT = "python"; public static final String API_AUTHENTICATE = "api.authenticate"; public static final String API_USE_SSL = "api.ssl"; public static final String API_CSRF_PREVENTION_KEY = "api.csrfPrevention.enabled"; @@ -237,6 +239,10 @@ public class Configuration { public static final String DEFAULT_SCHEDULER_START_DELAY_SECONDS = "120"; public static final String SERVER_TMP_DIR_KEY = "server.tmp.dir"; public static final String SERVER_TMP_DIR_DEFAULT = "/var/lib/ambari-server/tmp"; + + public static final String DEF_ARCHIVE_EXTENSION; + public static final String DEF_ARCHIVE_CONTENT_TYPE; + /** * This key defines whether stages of parallel requests are executed in * parallel or sequentally. Only stages from different requests @@ -328,6 +334,17 @@ public class Configuration { private volatile boolean credentialProviderInitialized = false; private Map<String, String> customDbProperties = null; + static { + if (System.getProperty("os.name").contains("Windows")) { + DEF_ARCHIVE_EXTENSION = ".zip"; + DEF_ARCHIVE_CONTENT_TYPE = "application/zip"; + } + else { + DEF_ARCHIVE_EXTENSION = ".tar.gz"; + DEF_ARCHIVE_CONTENT_TYPE = "application/x-ustar"; + } + } + public Configuration() { this(readConfigFile()); } @@ -342,6 +359,8 @@ public class Configuration { this.properties = properties; configsMap = new HashMap<String, String>(); + configsMap.put(AMBARI_PYTHON_WRAP_KEY, properties.getProperty( + AMBARI_PYTHON_WRAP_KEY, AMBARI_PYTHON_WRAP_DEFAULT)); configsMap.put(SRVR_TWO_WAY_SSL_KEY, properties.getProperty( SRVR_TWO_WAY_SSL_KEY, SRVR_TWO_WAY_SSL_DEFAULT)); configsMap.put(SRVR_TWO_WAY_SSL_PORT_KEY, properties.getProperty( http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java index 312f23f..1cfc1d0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java @@ -123,6 +123,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv Configuration configs = new Configuration(); String TMP_PATH = configs.getProperty(Configuration.SERVER_TMP_DIR_KEY); + String pythonCmd = configs.getProperty(Configuration.AMBARI_PYTHON_WRAP_KEY); AmbariManagementController managementController = getManagementController(); ConfigHelper configHelper = managementController.getConfigHelper(); Cluster cluster = null; @@ -277,7 +278,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv commandParams.put("xml_configs_list", xmlConfigs); commandParams.put("env_configs_list", envConfigs); - commandParams.put("output_file", componentName + "-configs.tar.gz"); + commandParams.put("output_file", componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION); Map<String, Object> jsonContent = new TreeMap<String, Object>(); jsonContent.put("configurations", configurations); @@ -308,7 +309,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv throw new SystemException("Failed to write configurations to json file ", e); } - String cmd = "ambari-python-wrap " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " + + String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " + packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json" + " INFO " + TMP_PATH; try { @@ -331,6 +332,8 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv throw new SystemException("Controller error ", e); } + Resource resource = new ResourceImpl(Resource.Type.ClientConfig); + resources.add(resource); return resources; }