Repository: ambari Updated Branches: refs/heads/trunk 41cf385ff -> d256ab8f9
AMBARI-8174. Ambari-deployed cluster can't start datanode as root from command line. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d256ab8f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d256ab8f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d256ab8f Branch: refs/heads/trunk Commit: d256ab8f94e9dc1292c9ce69e76235b215ca8dc0 Parents: 41cf385 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Thu Nov 6 19:11:39 2014 -0800 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Thu Nov 6 19:11:39 2014 -0800 ---------------------------------------------------------------------- .../python/resource_management/core/source.py | 3 +- .../2.0.6/services/HDFS/package/scripts/hdfs.py | 22 ------- .../services/HDFS/package/scripts/utils.py | 18 ++++-- .../services/HDFS/configuration/hadoop-env.xml | 7 +-- .../python/stacks/2.0.6/HDFS/test_datanode.py | 64 +++----------------- 5 files changed, 27 insertions(+), 87 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d256ab8f/ambari-common/src/main/python/resource_management/core/source.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/source.py b/ambari-common/src/main/python/resource_management/core/source.py index 22e1c6d..bff22f3 100644 --- a/ambari-common/src/main/python/resource_management/core/source.py +++ b/ambari-common/src/main/python/resource_management/core/source.py @@ -114,7 +114,8 @@ else: self.context = variables.copy() if variables else {} if not hasattr(self, 'template_env'): self.template_env = JinjaEnvironment(loader=TemplateLoader(self.env), - autoescape=False, undefined=StrictUndefined, trim_blocks=True) + autoescape=False, undefined=StrictUndefined, + trim_blocks=True) self.template = self.template_env.get_template(self.name) http://git-wip-us.apache.org/repos/asf/ambari/blob/d256ab8f/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py index c192682..25c1067 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py @@ -81,25 +81,3 @@ def hdfs(name=None): if params.lzo_enabled: Package(params.lzo_packages_for_current_host) - -def setup_hadoop_env(replace=False): - import params - - if params.security_enabled: - tc_owner = "root" - else: - tc_owner = params.hdfs_user - Directory(params.hadoop_conf_empty_dir, - recursive=True, - owner='root', - group='root' - ) - Link(params.hadoop_conf_dir, - to=params.hadoop_conf_empty_dir, - not_if=format("ls {hadoop_conf_dir}") - ) - File(os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh'), - owner=tc_owner, - content=InlineTemplate(params.hadoop_env_sh_template), - replace=replace - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/d256ab8f/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py index 150d0a4..0983c3f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py @@ -103,11 +103,22 @@ def service(action=None, name=None, user=None, create_pid_dir=False, pass pass + service_is_up = check_process if action == "start" else None + # Set HADOOP_SECURE_DN_USER correctly in hadoop-env if DN is running as root # in secure mode. - if name == 'datanode' and user == 'root': - params.dn_proc_user = 'root' - hdfs.setup_hadoop_env(replace=True) + set_secure_dn_user_cmd="sed -i 's/export HADOOP_SECURE_DN_USER=.*/export " \ + "HADOOP_SECURE_DN_USER=\"{0}\"/' {1}" + if name == 'datanode' and action == 'start': + if user == 'root': + secure_dn_user = params.hdfs_user + else: + secure_dn_user = "" + pass + + Execute(set_secure_dn_user_cmd.format(secure_dn_user, + os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh')), + not_if=service_is_up) pass hadoop_env_exports_str = '' @@ -121,7 +132,6 @@ def service(action=None, name=None, user=None, create_pid_dir=False, daemon_cmd = format("{ulimit_cmd} su -s /bin/bash - {user} -c '{cmd} {action} {name}'") - service_is_up = check_process if action == "start" else None #remove pid file from dead process File(pid_file, action="delete", http://git-wip-us.apache.org/repos/asf/ambari/blob/d256ab8f/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml index d186922..b768cb4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml @@ -135,12 +135,7 @@ export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH} export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" -HDFS_DN_PROC_USER={{dn_proc_user}} -if [ $HDFS_DN_PROC_USER == "root" ]; then - export HADOOP_SECURE_DN_USER="{{hdfs_user}}" -else - export HADOOP_SECURE_DN_USER="" -fi +export HADOOP_SECURE_DN_USER="" </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/d256ab8f/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py index b376dfb..1b4b014 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py @@ -49,6 +49,9 @@ class TestDatanode(RMFTestCase): owner = 'hdfs', recursive = True, ) + self.assertResourceCalled('Execute', "sed -i 's/export HADOOP_SECURE_DN_USER=.*/export HADOOP_SECURE_DN_USER=\"\"/' /etc/hadoop/conf/hadoop-env.sh", + not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1', + ) self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid', action = ['delete'], not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1', @@ -109,19 +112,8 @@ class TestDatanode(RMFTestCase): owner = 'hdfs', recursive = True, ) - self.assertResourceCalled('Directory', '/etc/hadoop/conf.empty', - recursive=True, - owner='root', - group='root' - ) - self.assertResourceCalled('Link', '/etc/hadoop/conf', - to='/etc/hadoop/conf.empty', - not_if='ls /etc/hadoop/conf' - ) - self.assertResourceCalled('File', os.path.join('/etc/hadoop/conf', 'hadoop-env.sh'), - owner='root', - content=InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']), - replace=True + self.assertResourceCalled('Execute', "sed -i 's/export HADOOP_SECURE_DN_USER=.*/export HADOOP_SECURE_DN_USER=\"hdfs\"/' /etc/hadoop/conf/hadoop-env.sh", + not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1', ) self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid', action = ['delete'], @@ -153,19 +145,8 @@ class TestDatanode(RMFTestCase): owner = 'hdfs', recursive = True, ) - self.assertResourceCalled('Directory', '/etc/hadoop/conf.empty', - recursive=True, - owner='root', - group='root' - ) - self.assertResourceCalled('Link', '/etc/hadoop/conf', - to='/etc/hadoop/conf.empty', - not_if='ls /etc/hadoop/conf' - ) - self.assertResourceCalled('File', os.path.join('/etc/hadoop/conf', 'hadoop-env.sh'), - owner='root', - content=InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']), - replace=True + self.assertResourceCalled('Execute', "sed -i 's/export HADOOP_SECURE_DN_USER=.*/export HADOOP_SECURE_DN_USER=\"hdfs\"/' /etc/hadoop/conf/hadoop-env.sh", + not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1', ) self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid', action = ['delete'], @@ -200,6 +181,9 @@ class TestDatanode(RMFTestCase): owner = 'hdfs', recursive = True, ) + self.assertResourceCalled('Execute', "sed -i 's/export HADOOP_SECURE_DN_USER=.*/export HADOOP_SECURE_DN_USER=\"\"/' /etc/hadoop/conf/hadoop-env.sh", + not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1', + ) self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid', action = ['delete'], not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1', @@ -224,20 +208,6 @@ class TestDatanode(RMFTestCase): owner = 'hdfs', recursive = True, ) - self.assertResourceCalled('Directory', '/etc/hadoop/conf.empty', - recursive=True, - owner='root', - group='root' - ) - self.assertResourceCalled('Link', '/etc/hadoop/conf', - to='/etc/hadoop/conf.empty', - not_if='ls /etc/hadoop/conf' - ) - self.assertResourceCalled('File', os.path.join('/etc/hadoop/conf', 'hadoop-env.sh'), - owner='root', - content=InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']), - replace=True - ) self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid', action = ['delete'], not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1', @@ -272,20 +242,6 @@ class TestDatanode(RMFTestCase): owner = 'hdfs', recursive = True, ) - self.assertResourceCalled('Directory', '/etc/hadoop/conf.empty', - recursive=True, - owner='root', - group='root' - ) - self.assertResourceCalled('Link', '/etc/hadoop/conf', - to='/etc/hadoop/conf.empty', - not_if='ls /etc/hadoop/conf' - ) - self.assertResourceCalled('File', os.path.join('/etc/hadoop/conf', 'hadoop-env.sh'), - owner='root', - content=InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']), - replace=True - ) self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid', action = ['delete'], not_if='ls /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid` >/dev/null 2>&1',
