Repository: ambari Updated Branches: refs/heads/branch-1.7.0 71d2acc9f -> 315c1ca79
AMBARI-8081. Ambari Storm Security config changes. (mahadev) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/315c1ca7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/315c1ca7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/315c1ca7 Branch: refs/heads/branch-1.7.0 Commit: 315c1ca79555905dd84c8b759cb496cb89ad2f16 Parents: 71d2acc Author: Mahadev Konar <maha...@apache.org> Authored: Fri Oct 31 13:13:17 2014 -0700 Committer: Mahadev Konar <maha...@apache.org> Committed: Fri Oct 31 13:13:21 2014 -0700 ---------------------------------------------------------------------- .../services/STORM/package/scripts/params.py | 3 + .../2.1/services/STORM/package/scripts/storm.py | 28 ++++++- .../STORM/package/templates/client_jaas.conf.j2 | 23 ++++++ .../STORM/package/templates/storm.yaml.j2 | 2 +- .../STORM/package/templates/storm_jaas.conf.j2 | 2 - .../package/templates/worker-launcher.cfg.j2 | 19 +++++ .../services/STORM/configuration/storm-site.xml | 7 +- .../2.1/STORM/test_storm_jaas_configuration.py | 18 +++- .../stacks/2.1/STORM/test_storm_nimbus.py | 17 +--- .../stacks/2.1/STORM/test_storm_supervisor.py | 87 -------------------- 10 files changed, 95 insertions(+), 111 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/315c1ca7/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py index 99547d5..32cb60d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py @@ -77,6 +77,9 @@ if security_enabled: storm_ui_host = default("/clusterHostInfo/storm_ui_server_hosts", []) storm_ui_jaas_principal = _storm_ui_jaas_principal_name.replace('_HOST',storm_ui_host[0].lower()) + storm_bare_jaas_principal = _storm_principal_name.replace('_HOST','').replace('@'+kerberos_domain,'') + + _nimbus_principal_name = config['configurations']['storm-env']['nimbus_principal_name'] nimbus_jaas_principal = _nimbus_principal_name.replace('_HOST',nimbus_host.lower()) http://git-wip-us.apache.org/repos/asf/ambari/blob/315c1ca7/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/storm.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/storm.py index 8b0c094..5699e57 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/storm.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/storm.py @@ -21,6 +21,7 @@ limitations under the License. from resource_management import * from yaml_utils import escape_yaml_propetry import sys +from ambari_agent.AgentException import AgentException def storm(): import params @@ -48,6 +49,7 @@ def storm(): group=params.user_group ) + File(format("{conf_dir}/storm-env.sh"), owner=params.storm_user, content=InlineTemplate(params.storm_env_sh_template) @@ -56,4 +58,28 @@ def storm(): if params.security_enabled: TemplateConfig(format("{conf_dir}/storm_jaas.conf"), owner=params.storm_user - ) \ No newline at end of file + ) + if params.is_compatible_to_2_2_stack: + TemplateConfig(format("{conf_dir}/client_jaas.conf"), + owner=params.storm_user + ) + minRuid = configurations['_storm.min.ruid'] if configurations.has_key('_storm.min.ruid') else '' + + min_user_ruid = int(minRuid) if minRuid.isdigit() else _find_real_user_min_uid() + + File(format("{conf_dir}/worker-launcher.cfg"), + content=Template("worker-launcher.cfg.j2",min_user_ruid = min_user_ruid), + owner='root', + group=params.user_group + ) + + +''' +Finds minimal real user UID +''' +def _find_real_user_min_uid(): + with open('/etc/login.defs') as f: + for line in f: + if line.strip().startswith('UID_MIN') and len(line.split()) == 2 and line.split()[1].isdigit(): + return int(line.split()[1]) + raise AgentException ("Unable to find UID_MIN in file /etc/login.defs. Expecting format e.g.: 'UID_MIN 500'") http://git-wip-us.apache.org/repos/asf/ambari/blob/315c1ca7/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/client_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/client_jaas.conf.j2 new file mode 100644 index 0000000..cf78af4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/client_jaas.conf.j2 @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +StormClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="{{nimbus_bare_jaas_principal}}" +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/315c1ca7/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/storm.yaml.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/storm.yaml.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/storm.yaml.j2 index 8491457..82ff239 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/storm.yaml.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/storm.yaml.j2 @@ -41,7 +41,7 @@ storm.thrift.transport : "{% if security_enabled %}{{configurations['_storm.thri # storm.principal.tolocal: "backtype.storm.security.auth.KerberosPrincipalToLocal" -storm.zookeeper.superACL: "sasl:{{storm_user}}" +storm.zookeeper.superACL: "sasl:{{storm_bare_jaas_principal}}" java.security.auth.login.config: "{{conf_dir}}/storm_jaas.conf" nimbus.admins: - "{{storm_user}}" http://git-wip-us.apache.org/repos/asf/ambari/blob/315c1ca7/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/storm_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/storm_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/storm_jaas.conf.j2 index f5e3ccf..7c650df 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/storm_jaas.conf.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/storm_jaas.conf.j2 @@ -31,7 +31,6 @@ StormClient { storeKey=true useTicketCache=false serviceName="{{nimbus_bare_jaas_principal}}" - debug=true principal="{{storm_jaas_principal}}"; }; {% endif %} @@ -42,6 +41,5 @@ Client { storeKey=true useTicketCache=false serviceName="zookeeper" - debug=true principal="{{storm_jaas_principal}}"; }; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/315c1ca7/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/worker-launcher.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/worker-launcher.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/worker-launcher.cfg.j2 new file mode 100644 index 0000000..2228601 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/templates/worker-launcher.cfg.j2 @@ -0,0 +1,19 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +storm.worker-launcher.group={{user_group}} +min.user.id={{min_user_ruid}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/315c1ca7/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml index be959aa..376a83d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml @@ -84,5 +84,10 @@ <value>-Xmx768m _JAAS_PLACEHOLDER</value> <description>Childopts for Storm DRPC Java process.</description> </property> - + + <property> + <name>_storm.min.ruid</name> + <value>null</value> + <description>min.user.id is set to the first real user id on the system. If value is 'null' than default value will be taken from key UID_MIN of /etc/login.defs otherwise the specified value will be used for all hosts.</description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/315c1ca7/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py index ddca5bc..d8b5465 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_jaas_configuration.py @@ -42,15 +42,18 @@ class TestStormJaasConfiguration(TestStormBase): self.assert_configure_default() - def test_configure_secured(self): + @patch("storm._find_real_user_min_uid") + def test_configure_secured(self, find_real_user_max_pid): + find_real_user_max_pid.return_value = 500 self.executeScript("2.1/services/STORM/package/scripts/nimbus.py", classname = "Nimbus", command = "configure", config_file = "secured-storm-start.json" ) self.assert_configure_secured() - - def test_start_secured(self): + @patch("storm._find_real_user_min_uid") + def test_start_secured(self, find_real_user_max_pid): + find_real_user_max_pid.return_value = 500 self.executeScript("2.1/services/STORM/package/scripts/nimbus.py", classname = "Nimbus", command = "start", @@ -68,7 +71,16 @@ class TestStormJaasConfiguration(TestStormBase): def assert_configure_secured(self): + storm_yarn_content = super(TestStormJaasConfiguration, self).assert_configure_secured() + self.assertResourceCalled('TemplateConfig', '/etc/storm/conf/client_jaas.conf', + owner = 'storm', + ) + self.assertResourceCalled('File', '/etc/storm/conf/worker-launcher.cfg', + owner = 'root', + content = Template('worker-launcher.cfg.j2', min_user_ruid = 500), + group = 'hadoop', + ) self.assertTrue(storm_yarn_content.find('_JAAS_PLACEHOLDER') == -1, 'Placeholder have to be substituted') self.assertTrue(storm_yarn_content.find('_storm') == -1, 'pairs start with _strom has to be removed') http://git-wip-us.apache.org/repos/asf/ambari/blob/315c1ca7/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py index 9ab9730..90c52a1 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py @@ -123,19 +123,4 @@ class TestStormNimbus(TestStormBase): ) self.assertResourceCalled('Execute', 'rm -f /var/run/storm/nimbus.pid') self.assertNoMoreResources() - - -# def call_storm_template_and_assert(self): -# import yaml_utils -# storm_yarn_template = Template( -# "storm.yaml.j2", -# extra_imports=[yaml_utils.escape_yaml_propetry], -# configurations = self.getConfig()['configurations']['storm-site']) -# storm_yarn_content = storm_yarn_template.get_content() -# -# self.assertResourceCalled('File', '/etc/storm/conf/storm.yaml', -# owner = 'storm', -# content= storm_yarn_template, -# group = 'hadoop' -# ) -# return storm_yarn_content + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/315c1ca7/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py index 1e0972f..806394b 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py @@ -166,90 +166,3 @@ class TestStormSupervisor(TestStormBase): self.assertResourceCalled('Execute', 'rm -f /var/run/storm/logviewer.pid') self.assertNoMoreResources() - def assert_configure_default(self): - - self.assertResourceCalled('Directory', '/var/log/storm', - owner = 'storm', - group = 'hadoop', - recursive = True, - ) - self.assertResourceCalled('Directory', '/var/run/storm', - owner = 'storm', - group = 'hadoop', - recursive = True, - ) - self.assertResourceCalled('Directory', '/hadoop/storm', - owner = 'storm', - group = 'hadoop', - recursive = True, - ) - self.assertResourceCalled('Directory', '/etc/storm/conf', - owner = 'storm', - group = 'hadoop', - recursive = True, - ) - self.assertResourceCalled('File', '/etc/storm/conf/config.yaml', - owner = 'storm', - content = Template('config.yaml.j2'), - group = 'hadoop', - ) - #assert that storm.yam was properly configured - self.call_storm_template_and_assert() - self.assertResourceCalled('File', '/etc/storm/conf/storm-env.sh', - owner = 'storm', - content = InlineTemplate(self.getConfig()['configurations']['storm-env']['content']) - ) - - def assert_configure_secured(self): - self.assertResourceCalled('Directory', '/var/log/storm', - owner = 'storm', - group = 'hadoop', - recursive = True, - ) - self.assertResourceCalled('Directory', '/var/run/storm', - owner = 'storm', - group = 'hadoop', - recursive = True, - ) - self.assertResourceCalled('Directory', '/hadoop/storm', - owner = 'storm', - group = 'hadoop', - recursive = True, - ) - self.assertResourceCalled('Directory', '/etc/storm/conf', - owner = 'storm', - group = 'hadoop', - recursive = True, - ) - self.assertResourceCalled('File', '/etc/storm/conf/config.yaml', - owner = 'storm', - content = Template('config.yaml.j2'), - group = 'hadoop', - ) -# self.assertResourceCalled('File', '/etc/storm/conf/storm.yaml', -# owner = 'storm', -# content = self.get_yaml_inline_template(self.getConfig()['configurations']['storm-site']), -# group = 'hadoop', -# mode = None, -# ) - #assert that storm.yam was properly configured - self.call_storm_template_and_assert() - - self.assertResourceCalled('File', '/etc/storm/conf/storm-env.sh', - owner = 'storm', - content = InlineTemplate(self.getConfig()['configurations']['storm-env']['content']) - ) - self.assertResourceCalled('TemplateConfig', '/etc/storm/conf/storm_jaas.conf', - owner = 'storm', - ) - -# def get_yaml_inline_template(self, configurations): -# with self.env: -# from yaml_config import yaml_inline_template -# from storm import enrich_kerberos_settings -# import params -# -# if params.security_enabled: -# return yaml_inline_template(enrich_kerberos_settings(configurations, params.conf_dir, "EXAMPLE.COM")) -# else: -# return yaml_inline_template(configurations)
