Repository: ambari Updated Branches: refs/heads/branch-2.5 c91ad2b37 -> 972b23fe2
AMBARI-19269. Zookeeper and RM connection is not secure. (Attila Magyar via stoader) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/972b23fe Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/972b23fe Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/972b23fe Branch: refs/heads/branch-2.5 Commit: 972b23fe224fb918608c460cc57c83fbb03298ef Parents: c91ad2b Author: Attila Magyar <amag...@hortonworks.com> Authored: Fri Dec 23 20:15:00 2016 +0100 Committer: Toader, Sebastian <stoa...@hortonworks.com> Committed: Fri Dec 23 20:15:00 2016 +0100 ---------------------------------------------------------------------- .../YARN/2.1.0.2.0/kerberos.json | 3 ++- .../2.1.0.2.0/package/scripts/params_linux.py | 7 ++++++ .../package/scripts/resourcemanager.py | 17 ++++++++++--- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 5 ++++ .../package/templates/yarn_jaas.conf.j2 | 26 ++++++++++++++++++++ .../stacks/HDP/2.2/services/YARN/kerberos.json | 3 ++- .../HDP/2.3.ECS/services/YARN/kerberos.json | 3 ++- .../stacks/HDP/2.3/services/YARN/kerberos.json | 3 ++- .../stacks/HDP/2.5/services/YARN/kerberos.json | 3 ++- .../stacks/2.0.6/YARN/test_historyserver.py | 8 +++++- .../stacks/2.0.6/YARN/test_mapreduce2_client.py | 8 +++++- .../stacks/2.0.6/YARN/test_nodemanager.py | 8 +++++- .../stacks/2.0.6/YARN/test_resourcemanager.py | 8 +++++- .../stacks/2.0.6/YARN/test_yarn_client.py | 8 +++++- 14 files changed, 97 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json index 4093431..a8379ee 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json @@ -31,7 +31,8 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "" + "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", + "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 0496995..aa0c4f5 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -171,6 +171,7 @@ rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager. rm_nodes_exclude_dir = os.path.dirname(rm_nodes_exclude_path) java64_home = config['hostLevelParams']['java_home'] +java_exec = format("{java64_home}/bin/java") hadoop_ssl_enabled = default("/configurations/core-site/hadoop.ssl.enabled", False) yarn_heapsize = config['configurations']['yarn-env']['yarn_heapsize'] @@ -244,11 +245,17 @@ rm_kinit_cmd = "" yarn_timelineservice_kinit_cmd = "" nodemanager_kinit_cmd = "" +rm_zk_address = config['configurations']['yarn-site']['yarn.resourcemanager.zk-address'] +rm_zk_znode = config['configurations']['yarn-site']['yarn.resourcemanager.zk-state-store.parent-path'] +rm_zk_store_class = config['configurations']['yarn-site']['yarn.resourcemanager.store.class'] + if security_enabled: rm_principal_name = config['configurations']['yarn-site']['yarn.resourcemanager.principal'] rm_principal_name = rm_principal_name.replace('_HOST',hostname.lower()) rm_keytab = config['configurations']['yarn-site']['yarn.resourcemanager.keytab'] rm_kinit_cmd = format("{kinit_path_local} -kt {rm_keytab} {rm_principal_name};") + yarn_jaas_file = os.path.join(config_dir, 'yarn_jaas.conf') + yarn_env_sh_template += format('\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config={yarn_jaas_file} -Dzookeeper.sasl.clientconfig=Client"\n') # YARN timeline security options if has_ats: http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py index 6a7eea7..7b887a3 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py @@ -38,7 +38,7 @@ from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil from resource_management.libraries.providers.hdfs_resource import HdfsResourceProvider from resource_management import is_empty from resource_management import shell - +from resource_management.core.resources.zkmigrator import ZkMigrator from yarn import yarn from service import service @@ -226,8 +226,19 @@ class ResourcemanagerDefault(Resourcemanager): pass pass - - + def disable_security(self, env): + import params + if 'ZKRMStateStore' not in params.rm_zk_store_class: + Logger.info("Skipping reverting ACL") + return + zkmigrator = ZkMigrator( + params.rm_zk_address, \ + params.java_exec, \ + params.java64_home, \ + params.yarn_jaas_file, \ + params.yarn_user) + Logger.info("Reverting ACL of znode %s" % params.rm_zk_znode) + zkmigrator.set_acls(params.rm_zk_znode, 'world:anyone:crdwa') def wait_for_dfs_directories_created(self, *dirs): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index f5acb11..81180f9 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -422,6 +422,11 @@ def yarn(name=None, config_dir=None): group = params.mapred_tt_group, content=Template("taskcontroller.cfg.j2") ) + File(os.path.join(config_dir, 'yarn_jaas.conf'), + owner=params.yarn_user, + group=params.user_group, + content=Template("yarn_jaas.conf.j2") + ) else: File(os.path.join(config_dir, 'taskcontroller.cfg'), owner=tc_owner, http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn_jaas.conf.j2 new file mode 100644 index 0000000..483c815 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="{{rm_keytab}}" + principal="{{rm_principal_name}}"; +}; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json index 2fdce8a..784589c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json @@ -32,7 +32,8 @@ "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", - "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore-secure" + "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore-secure", + "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json index b02b3e9..74b5746 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json @@ -34,7 +34,8 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "" + "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", + "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json index 0d67e59..c20bd23 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json @@ -32,7 +32,8 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "" + "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", + "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json index e690204..4cb18a9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json @@ -32,7 +32,8 @@ "yarn.resourcemanager.proxyusers.*.hosts": "", "yarn.resourcemanager.proxyusers.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "" + "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "", + "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py index bea3f81..92e2e29 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py @@ -674,7 +674,8 @@ class TestHistoryServer(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']), + content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content'] + + '\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config=/usr/hdp/current/hadoop-client/conf/yarn_jaas.conf -Dzookeeper.sasl.clientconfig=Client"\n'), owner = 'yarn', group = 'hadoop', mode = 0755, @@ -710,6 +711,11 @@ class TestHistoryServer(RMFTestCase): group = 'hadoop', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn_jaas.conf', + content = Template('yarn_jaas.conf.j2'), + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', owner = 'mapred', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py index 466b0f7..774f3c6 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py @@ -305,7 +305,8 @@ class TestMapReduce2Client(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']), + content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content'] + + '\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config=/usr/hdp/current/hadoop-client/conf/yarn_jaas.conf -Dzookeeper.sasl.clientconfig=Client"\n'), owner = 'yarn', group = 'hadoop', mode = 0755, @@ -341,6 +342,11 @@ class TestMapReduce2Client(RMFTestCase): group = 'hadoop', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn_jaas.conf', + content = Template('yarn_jaas.conf.j2'), + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', owner = 'mapred', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py index 4abf2c9..0eb5561 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py @@ -487,7 +487,8 @@ class TestNodeManager(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']), + content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content'] + + '\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config=/usr/hdp/current/hadoop-client/conf/yarn_jaas.conf -Dzookeeper.sasl.clientconfig=Client"\n'), owner = 'yarn', group = 'hadoop', mode = 0755, @@ -523,6 +524,11 @@ class TestNodeManager(RMFTestCase): group = 'hadoop', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn_jaas.conf', + content = Template('yarn_jaas.conf.j2'), + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', owner = 'mapred', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index 7b5ce18..da93096 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -459,7 +459,8 @@ class TestResourceManager(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']), + content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content'] + + '\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config=/usr/hdp/current/hadoop-client/conf/yarn_jaas.conf -Dzookeeper.sasl.clientconfig=Client"\n'), owner = 'yarn', group = 'hadoop', mode = 0755, @@ -495,6 +496,11 @@ class TestResourceManager(RMFTestCase): group = 'hadoop', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn_jaas.conf', + content = Template('yarn_jaas.conf.j2'), + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', owner = 'mapred', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/972b23fe/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py index 3719fe5..d4341e1 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py @@ -305,7 +305,8 @@ class TestYarnClient(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']), + content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content'] + + '\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config=/usr/hdp/current/hadoop-client/conf/yarn_jaas.conf -Dzookeeper.sasl.clientconfig=Client"\n'), owner = 'yarn', group = 'hadoop', mode = 0755, @@ -341,6 +342,11 @@ class TestYarnClient(RMFTestCase): group = 'hadoop', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/yarn_jaas.conf', + content = Template('yarn_jaas.conf.j2'), + owner = 'yarn', + group = 'hadoop', + ) self.assertResourceCalled('XmlConfig', 'mapred-site.xml', owner = 'mapred', group = 'hadoop',
