AMBARI-22160. hadooplzo package installation failed on devdeploys (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e19db403 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e19db403 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e19db403 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: e19db403e8a3320a5b47503e43e2a35277149a3f Parents: e037a8d Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Tue Oct 10 15:32:10 2017 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Tue Oct 10 15:32:10 2017 +0300 ---------------------------------------------------------------------- .../python/resource_management/libraries/script/script.py | 4 ++-- .../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 10 ++++++---- .../HDFS/2.1.0.2.0/package/scripts/install_params.py | 6 ------ .../HDFS/2.1.0.2.0/package/scripts/params_linux.py | 2 -- .../HDFS/3.0.0.3.0/package/scripts/hdfs.py | 10 ++++++---- .../HDFS/3.0.0.3.0/package/scripts/install_params.py | 6 ------ .../HDFS/3.0.0.3.0/package/scripts/params_linux.py | 2 -- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 6 ++++-- .../OOZIE/4.0.0.2.0/package/scripts/params_linux.py | 3 --- .../OOZIE/4.2.0.3.0/package/scripts/oozie.py | 5 +++-- .../OOZIE/4.2.0.3.0/package/scripts/params_linux.py | 3 --- 11 files changed, 21 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 4282213..bf8c0dc 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -778,10 +778,10 @@ class Script(object): pkg_provider = get_provider("Package") try: - available_packages_in_repos = pkg_provider.get_available_packages_in_repos(self.get_config()['repositoryFile']['repositories']) + self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(self.get_config()['repositoryFile']['repositories']) except Exception as err: Logger.exception("Unable to load available packages") - available_packages_in_repos = [] + self.available_packages_in_repos = [] def install_packages(self, env): http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py index e054209..07c7616 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py @@ -25,6 +25,7 @@ from resource_management.core.resources import Package from resource_management.core.source import Template from resource_management.core.resources.service import ServiceConfig from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages import os from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst @@ -138,10 +139,11 @@ def hdfs(name=None): content=Template("slaves.j2") ) - if params.lzo_enabled and len(params.lzo_packages) > 0: - Package(params.lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) + if params.lzo_enabled: + lzo_packages = get_lzo_packages(params.stack_version_unformatted) + Package(lzo_packages, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, + retry_count=params.agent_stack_retry_count) def install_snappy(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py index fe488c3..235f231 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py @@ -23,7 +23,6 @@ if OSCheck.is_windows_family(): exclude_packages = [] else: from resource_management.libraries.functions.default import default - from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.script.script import Script _config = Script.get_config() @@ -32,8 +31,3 @@ else: # The logic for LZO also exists in OOZIE's params.py io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - lzo_packages = get_lzo_packages(stack_version_unformatted) - - exclude_packages = [] - if not lzo_enabled: - exclude_packages += lzo_packages http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 76b430b..bb6349b 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -40,7 +40,6 @@ from resource_management.libraries.functions.get_not_managed_resources import ge from resource_management.libraries.script.script import Script from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions.format_jvm_option import format_jvm_option -from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs from resource_management.libraries.functions import is_empty from resource_management.libraries.functions.get_architecture import get_architecture @@ -389,7 +388,6 @@ HdfsResource = functools.partial( # The logic for LZO also exists in OOZIE's params.py io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() -lzo_packages = get_lzo_packages(stack_version_unformatted) name_node_params = default("/commandParams/namenode", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py index e054209..07c7616 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py @@ -25,6 +25,7 @@ from resource_management.core.resources import Package from resource_management.core.source import Template from resource_management.core.resources.service import ServiceConfig from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages import os from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst @@ -138,10 +139,11 @@ def hdfs(name=None): content=Template("slaves.j2") ) - if params.lzo_enabled and len(params.lzo_packages) > 0: - Package(params.lzo_packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) + if params.lzo_enabled: + lzo_packages = get_lzo_packages(params.stack_version_unformatted) + Package(lzo_packages, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, + retry_count=params.agent_stack_retry_count) def install_snappy(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py index fe488c3..235f231 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py @@ -23,7 +23,6 @@ if OSCheck.is_windows_family(): exclude_packages = [] else: from resource_management.libraries.functions.default import default - from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.script.script import Script _config = Script.get_config() @@ -32,8 +31,3 @@ else: # The logic for LZO also exists in OOZIE's params.py io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - lzo_packages = get_lzo_packages(stack_version_unformatted) - - exclude_packages = [] - if not lzo_enabled: - exclude_packages += lzo_packages http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py index de735f4..2fa6208 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py @@ -40,7 +40,6 @@ from resource_management.libraries.functions.get_not_managed_resources import ge from resource_management.libraries.script.script import Script from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions.format_jvm_option import format_jvm_option -from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs from resource_management.libraries.functions import is_empty from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config @@ -378,7 +377,6 @@ HdfsResource = functools.partial( # The logic for LZO also exists in OOZIE's params.py io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() -lzo_packages = get_lzo_packages(stack_version_unformatted) name_node_params = default("/commandParams/namenode", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index 64f9d54..f215a1e 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -37,6 +37,7 @@ from resource_management.libraries.functions.copy_tarball import get_current_ver from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.libraries.script.script import Script from resource_management.libraries.functions.security_commons import update_credential_provider_path +from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.core.resources.packaging import Package from resource_management.core.shell import as_user, as_sudo, call, checked_call from resource_management.core.exceptions import Fail @@ -305,8 +306,9 @@ def oozie_server_specific(upgrade_type): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) - if params.lzo_enabled and len(params.all_lzo_packages) > 0: - Package(params.all_lzo_packages, + if params.lzo_enabled: + all_lzo_packages = get_lzo_packages(params.stack_version_unformatted) + Package(all_lzo_packages, retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index b66e157..a0f0672 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -30,7 +30,6 @@ from resource_management.libraries.functions import get_port_from_url from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.libraries.script.script import Script -from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.expect import expect from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions.get_architecture import get_architecture @@ -388,5 +387,3 @@ is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled' # The logic for LZO also exists in HDFS' params.py io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - -all_lzo_packages = get_lzo_packages(stack_version_unformatted) http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py index d916d3b..0771e93 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py @@ -275,8 +275,9 @@ def oozie_server_specific(): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) - if params.lzo_enabled and len(params.all_lzo_packages) > 0: - Package(params.all_lzo_packages, + if params.lzo_enabled: + all_lzo_packages = get_lzo_packages(params.stack_version_unformatted) + Package(all_lzo_packages, retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py index d30a465..70b89b7 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py @@ -28,7 +28,6 @@ from resource_management.libraries.functions import get_port_from_url from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.libraries.script.script import Script -from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.expect import expect from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions.get_architecture import get_architecture @@ -370,5 +369,3 @@ is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled' # The logic for LZO also exists in HDFS' params.py io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - -all_lzo_packages = get_lzo_packages(stack_version_unformatted)