Repository: ambari Updated Branches: refs/heads/trunk db09f15f9 -> ce429e750
AMBARI-11476. RU from Dal to Dal+ has failures in First Oozie Service Check, then Oozie Server Start, because yarn-site.xml missing from hadoop conf and conf folder is incorrect (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ce429e75 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ce429e75 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ce429e75 Branch: refs/heads/trunk Commit: ce429e75014edfb08ea20dbff3826127ab3a509a Parents: db09f15 Author: Alejandro Fernandez <afernan...@hortonworks.com> Authored: Wed May 27 19:49:11 2015 -0700 Committer: Alejandro Fernandez <afernan...@hortonworks.com> Committed: Thu May 28 14:21:07 2015 -0700 ---------------------------------------------------------------------- .../libraries/functions/copy_tarball.py | 24 ++++++++++---------- .../package/scripts/oozie_server_upgrade.py | 2 +- .../4.0.0.2.0/package/scripts/service_check.py | 20 ++++++++++++++-- .../stacks/2.0.6/OOZIE/test_service_check.py | 21 +++++++++++++++-- .../stacks/2.0.6/YARN/test_resourcemanager.py | 4 ++-- 5 files changed, 52 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ce429e75/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py index acb1e9f..f8a2570 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py @@ -31,23 +31,23 @@ STACK_VERSION_PATTERN = "{{ stack_version }}" TARBALL_MAP = { "HDP": { - "tez": ("/usr/hdp/%s/tez/lib/tez.tar.gz" % STACK_VERSION_PATTERN, - "/hdp/apps/%s/tez/tez.tar.gz" % STACK_VERSION_PATTERN), + "tez": ("/usr/hdp/{0}/tez/lib/tez.tar.gz".format(STACK_VERSION_PATTERN), + "/hdp/apps/{0}/tez/tez.tar.gz".format(STACK_VERSION_PATTERN)), - "hive": ("/usr/hdp/%s/hive/hive.tar.gz" % STACK_VERSION_PATTERN, - "/hdp/apps/%s/hive/hive.tar.gz" % STACK_VERSION_PATTERN), + "hive": ("/usr/hdp/{0}/hive/hive.tar.gz".format(STACK_VERSION_PATTERN), + "/hdp/apps/{0}/hive/hive.tar.gz".format(STACK_VERSION_PATTERN)), - "pig": ("/usr/hdp/%s/pig/pig.tar.gz" % STACK_VERSION_PATTERN, - "/hdp/apps/%s/pig/pig.tar.gz" % STACK_VERSION_PATTERN), + "pig": ("/usr/hdp/{0}/pig/pig.tar.gz".format(STACK_VERSION_PATTERN), + "/hdp/apps/{0}/pig/pig.tar.gz".format(STACK_VERSION_PATTERN)), - "hadoop_streaming": ("/usr/hdp/%s/hadoop/hadoop-streaming.jar" % STACK_VERSION_PATTERN, - "/hdp/apps/%s/mapreduce/hadoop-streaming.jar" % STACK_VERSION_PATTERN), + "hadoop_streaming": ("/usr/hdp/{0}/hadoop-mapreduce/hadoop-streaming.jar".format(STACK_VERSION_PATTERN), + "/hdp/apps/{0}/mapreduce/hadoop-streaming.jar".format(STACK_VERSION_PATTERN)), - "sqoop": ("/usr/hdp/%s/sqoop/sqoop.tar.gz" % STACK_VERSION_PATTERN, - "/hdp/apps/%s/sqoop/sqoop.tar.gz" % STACK_VERSION_PATTERN), + "sqoop": ("/usr/hdp/{0}/sqoop/sqoop.tar.gz".format(STACK_VERSION_PATTERN), + "/hdp/apps/{0}/sqoop/sqoop.tar.gz".format(STACK_VERSION_PATTERN)), - "mapreduce": ("/usr/hdp/%s/hadoop/mapreduce.tar.gz" % STACK_VERSION_PATTERN, - "/hdp/apps/%s/mapreduce/mapreduce.tar.gz" % STACK_VERSION_PATTERN) + "mapreduce": ("/usr/hdp/{0}/hadoop/mapreduce.tar.gz".format(STACK_VERSION_PATTERN), + "/hdp/apps/{0}/mapreduce/mapreduce.tar.gz".format(STACK_VERSION_PATTERN)) } } http://git-wip-us.apache.org/repos/asf/ambari/blob/ce429e75/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py index 47361c4..1afae6a 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py @@ -222,4 +222,4 @@ def _get_directory_mappings(): """ import params - return { params.conf_dir : BACKUP_CONF_ARCHIVE } + return { params.conf_dir + "/" : BACKUP_CONF_ARCHIVE } # the trailing "/" is important here so as to not include the "conf" folder itself http://git-wip-us.apache.org/repos/asf/ambari/blob/ce429e75/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py index 11f4431..2438543 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py @@ -17,16 +17,21 @@ See the License for the specific language governing permissions and limitations under the License. """ +import os +import glob + from resource_management.core.resources.system import Execute from resource_management.core.resources import File from resource_management.core.source import StaticFile from resource_management.core.system import System from resource_management.libraries.functions import format from resource_management.libraries.script import Script +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.core.exceptions import Fail from ambari_commons.os_family_impl import OsFamilyImpl from ambari_commons import OSConst -import os -import glob + +from resource_management.core.logger import Logger class OozieServiceCheck(Script): pass @@ -42,6 +47,17 @@ class OozieServiceCheckDefault(OozieServiceCheck): prepare_hdfs_file_name = 'prepareOozieHdfsDirectories.sh' smoke_test_file_name = 'oozieSmoke2.sh' + if 'yarn-site' in params.config['configurations']: + XmlConfig("yarn-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['yarn-site'], + owner=params.hdfs_user, + group=params.user_group, + mode=0644 + ) + else: + raise Fail("yarn-site.xml was not present in config parameters.") + OozieServiceCheckDefault.oozie_smoke_shell_file(smoke_test_file_name, prepare_hdfs_file_name) @staticmethod http://git-wip-us.apache.org/repos/asf/ambari/blob/ce429e75/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py index 219c874..ffa47cb 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py @@ -35,11 +35,20 @@ class TestServiceCheck(RMFTestCase): hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) - + + self.assertResourceCalled('XmlConfig', 'yarn-site.xml', + owner = 'hdfs', + group = 'hadoop', + conf_dir = '/etc/hadoop/conf', + mode = 0644, + configurations = self.getConfig()['configurations']['yarn-site'], + ) + self.assert_service_check() self.assertNoMoreResources() def test_service_check_secured(self): + self.maxDiff = None self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py", classname="OozieServiceCheck", command="service_check", @@ -47,7 +56,15 @@ class TestServiceCheck(RMFTestCase): hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) - + + self.assertResourceCalled('XmlConfig', 'yarn-site.xml', + owner = 'hdfs', + group = 'hadoop', + conf_dir = '/etc/hadoop/conf', + mode = 0644, + configurations = self.getConfig()['configurations']['yarn-site'], + ) + self.assert_service_check() self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/ce429e75/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index e7957fc..448fc01 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -185,7 +185,7 @@ class TestResourceManager(RMFTestCase): content = Template('exclude_hosts_list.j2'), group = 'hadoop', ) - self.assertResourceCalled('Execute', '{rm_kinit_cmd} yarn --config {conf_dir} rmadmin -refreshNodes', + self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes', environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"}, user = 'yarn', ) @@ -204,7 +204,7 @@ class TestResourceManager(RMFTestCase): content = Template('exclude_hosts_list.j2'), group = 'hadoop', ) - self.assertResourceCalled('Execute', '{rm_kinit_cmd} yarn --config {conf_dir} rmadmin -refreshNodes', + self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab rm/c6401.ambari.apache....@example.com; yarn --config /etc/hadoop/conf rmadmin -refreshNodes', environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"}, user = 'yarn', )