Repository: ambari Updated Branches: refs/heads/trunk 90a88c6aa -> 008cc6bad
AMBARI-10200. Storm secure config not visible through Ambari storm config UI (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/008cc6ba Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/008cc6ba Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/008cc6ba Branch: refs/heads/trunk Commit: 008cc6bad32f054de1686af0921ca0a2d024fa5a Parents: 90a88c6 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Wed Mar 25 17:46:08 2015 +0200 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Wed Mar 25 17:46:08 2015 +0200 ---------------------------------------------------------------------- .../0.1.0/configuration/storm-site.xml | 34 ++++++++++ .../AMBARI_METRICS/0.1.0/metainfo.xml | 4 ++ .../0.9.1.2.1/configuration/storm-site.xml | 5 ++ .../STORM/0.9.1.2.1/kerberos.json | 7 ++- .../STORM/0.9.1.2.1/package/scripts/params.py | 6 ++ .../STORM/0.9.1.2.1/package/scripts/storm.py | 10 ++- .../0.9.1.2.1/package/scripts/yaml_utils.py | 27 ++++++-- .../0.9.1.2.1/package/templates/storm.yaml.j2 | 65 -------------------- .../services/STORM/configuration/storm-site.xml | 6 +- .../services/STORM/configuration/storm-site.xml | 6 +- .../python/stacks/2.1/STORM/test_storm_base.py | 29 +++++---- .../stacks/2.1/configs/default-storm-start.json | 1 + .../stacks/2.1/configs/secured-storm-start.json | 3 +- 13 files changed, 103 insertions(+), 100 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml new file mode 100644 index 0000000..35456d7 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml @@ -0,0 +1,34 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="true"> + <property> + <name>topology.metrics.consumer.register</name> + <value>[{'class': 'org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink', 'parallelism.hint': 1}]</value> + <description>Topology metrics consumer register parameters.</description> + </property> + <property> + <name>metrics.reporter.register</name> + <value>org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter</value> + <description>Topology metrics reporter.</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml index ac954e6..39b640c 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml @@ -129,6 +129,10 @@ <config-type>ams-hbase-log4j</config-type> </configuration-dependencies> + <excluded-config-types> + <config-type>storm-site</config-type> + </excluded-config-types> + </service> </services> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml index b6a2cf2..5e96fb1 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml @@ -37,6 +37,11 @@ have permission to read/write from this location.</description> </property> <property> + <name>storm.log.dir</name> + <value>{{log_dir}}</value> + <description>Log directory for Storm.</description> + </property> + <property> <name>storm.zookeeper.servers</name> <value>['localhost']</value> <description>A list of hosts of ZooKeeper servers used to manage the cluster.</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/kerberos.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/kerberos.json index 50b92ff..94eaae1 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/kerberos.json +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/kerberos.json @@ -37,7 +37,12 @@ "drpc.authorizer": "backtype.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer", "ui.filter": "org.apache.hadoop.security.authentication.server.AuthenticationFilter", "storm.principal.tolocal": "backtype.storm.security.auth.KerberosPrincipalToLocal", - "supervisor.enable": "true" + "supervisor.enable": "true", + "storm.zookeeper.superACL": "sasl:{{storm_bare_jaas_principal}}", + "java.security.auth.login.config": "{{conf_dir}}/storm_jaas.conf", + "nimbus.admins": "['{{storm_user}}']", + "nimbus.supervisor.users": "['{{storm_user}}']", + "ui.filter.params": "{'type': 'kerberos', 'kerberos.principal': '{{storm_ui_jaas_principal}}', 'kerberos.keytab': '{{storm_ui_keytab_path}}', 'kerberos.name.rules': 'DEFAULT'}" } } ], http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py index 0e432ba..2d575b0 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py @@ -111,6 +111,12 @@ if security_enabled: nimbus_bare_jaas_principal = get_bare_principal(_nimbus_principal_name) nimbus_keytab_path = config['configurations']['storm-env']['nimbus_keytab'] +if stack_is_hdp22_or_further: + if security_enabled: + storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.secure.transport'] + else: + storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.nonsecure.transport'] + ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", []) has_metric_collector = not len(ams_collector_hosts) == 0 if has_metric_collector: http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py index 7a2488d..3ae3f92 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py @@ -18,6 +18,7 @@ limitations under the License. """ +import resource_management from resource_management.core.resources import File from resource_management.core.resources import Execute from resource_management.core.resources import Directory @@ -26,7 +27,7 @@ from resource_management.libraries.resources.template_config import TemplateConf from resource_management.libraries.functions.format import format from resource_management.core.source import Template from resource_management.libraries.functions import compare_versions -from yaml_utils import escape_yaml_propetry +from yaml_utils import yaml_config_template def storm(): import params @@ -58,12 +59,9 @@ def storm(): ) configurations = params.config['configurations']['storm-site'] - + File(format("{conf_dir}/storm.yaml"), - content=Template( - "storm.yaml.j2", - extra_imports=[escape_yaml_propetry], - configurations = configurations), + content=yaml_config_template(configurations), owner=params.storm_user, group=params.user_group ) http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py index 8187666..0034123 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py @@ -19,6 +19,8 @@ limitations under the License. """ import re +import resource_management +from resource_management.core.source import InlineTemplate def escape_yaml_propetry(value): unquouted = False @@ -26,10 +28,10 @@ def escape_yaml_propetry(value): if value in unquouted_values: unquouted = True - # if is list [a,b,c] - if re.match('^\w*\[.+\]\w*$', value): + # if is list [a,b,c] or dictionary {a: v, b: v2, c: v3} + if re.match('^\w*\[.+\]\w*$', value) or re.match('^\w*\{.+\}\w*$', value): unquouted = True - + try: int(value) unquouted = True @@ -46,4 +48,21 @@ def escape_yaml_propetry(value): value = value.replace("'","''") value = "'"+value+"'" - return value \ No newline at end of file + return value + +def replace_jaas_placeholder(name, security_enabled, conf_dir): + if name.find('_JAAS_PLACEHOLDER') > -1: + if security_enabled: + return name.replace('_JAAS_PLACEHOLDER', '-Djava.security.auth.login.config=' + conf_dir + '/storm_jaas.conf') + else: + return name.replace('_JAAS_PLACEHOLDER', '') + else: + return name + +storm_yaml_template = """{% for key, value in configurations|dictsort if not key.startswith('_') %}{{key}} : {{ escape_yaml_propetry(replace_jaas_placeholder(resource_management.core.source.InlineTemplate(value).get_content().strip(), security_enabled, conf_dir)) }} +{% endfor %}""" + +def yaml_config_template(configurations): + return InlineTemplate(storm_yaml_template, configurations=configurations, + extra_imports=[escape_yaml_propetry, replace_jaas_placeholder, resource_management, + resource_management.core, resource_management.core.source]) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.yaml.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.yaml.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.yaml.j2 deleted file mode 100644 index cc3e631..0000000 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm.yaml.j2 +++ /dev/null @@ -1,65 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} -{% macro replace_jaas_placeholder(name) -%} -{% if name.find('_JAAS_PLACEHOLDER') > -1 -%} - {%- if security_enabled -%} - {{ name.replace('_JAAS_PLACEHOLDER', '-Djava.security.auth.login.config=' +conf_dir + '/storm_jaas.conf') }} - {%- else -%} - {{ name.replace('_JAAS_PLACEHOLDER', '') }} - {%- endif -%} -{%- else -%} - {{name}} -{%- endif -%} -{%- endmacro %} - -{% for key, value in configurations|dictsort if not key.startswith('_') %} -{{key}} : {{ escape_yaml_propetry(replace_jaas_placeholder(value)) }} -{% endfor %} - -{% if stack_is_hdp22_or_further %} -storm.thrift.transport : "{% if security_enabled %}{{configurations['_storm.thrift.secure.transport']}}{% else %}{{configurations['_storm.thrift.nonsecure.transport']}}{% endif %}" -{% endif %} - -{% if security_enabled and stack_is_hdp22_or_further %} -# -# Kerberos security section. For the reference please use: https://github.com/hortonworks/storm/blob/champlain/SECURITY.md for details -# - -storm.zookeeper.superACL: "sasl:{{storm_bare_jaas_principal}}" -java.security.auth.login.config: "{{conf_dir}}/storm_jaas.conf" -nimbus.admins: - - "{{storm_user}}" -nimbus.supervisor.users: - - "{{storm_user}}" - -ui.filter.params: - "type": "kerberos" - "kerberos.principal": "{{storm_ui_jaas_principal}}" - "kerberos.keytab": "{{storm_ui_keytab_path}}" - "kerberos.name.rules": "DEFAULT" - -{% endif %} - -{% if has_metric_collector %} -topology.metrics.consumer.register: - - class: "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink" - parallelism.hint: 1 -metrics.reporter.register: "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter" -{% endif %} - -storm.log.dir: "{{log_dir}}" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml index befb5f8..c933099 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/storm-site.xml @@ -24,10 +24,8 @@ <property> <name>storm.thrift.transport</name> - <value>--</value> - <deleted>true</deleted> - <description>This is calculated field that is based of security state and _storm.thrift.nonsecure.transport and _storm.thrift.secure.transport - </description> + <value>{{storm_thrift_transport}}</value> + <description>The transport plug-in that used for Thrift client/server communication.</description> </property> <property> <name>_storm.thrift.nonsecure.transport</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/STORM/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/STORM/configuration/storm-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/STORM/configuration/storm-site.xml index c0528dd..1a50e8f 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/STORM/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/STORM/configuration/storm-site.xml @@ -24,10 +24,8 @@ <property> <name>storm.thrift.transport</name> - <value>--</value> - <deleted>true</deleted> - <description>This is calculated field that is based of security state and _storm.thrift.nonsecure.transport and _storm.thrift.secure.transport - </description> + <value>{{storm_thrift_transport}}</value> + <description>The transport plug-in that used for Thrift client/server communication.</description> </property> <property> <name>_storm.thrift.nonsecure.transport</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py index 2b475d1..f7b634c 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py @@ -19,8 +19,8 @@ limitations under the License. ''' from mock.mock import MagicMock, call, patch -from stacks.utils.RMFTestCase import * import resource_management.core.source +from stacks.utils.RMFTestCase import * import re @@ -62,7 +62,7 @@ class TestStormBase(RMFTestCase): storm_yarn_content = self.call_storm_template_and_assert() self.assertTrue(storm_yarn_content.find('_JAAS_PLACEHOLDER') == -1, 'Placeholder have to be substituted') - + self.assertResourceCalled('File', '/etc/storm/conf/storm-env.sh', owner = 'storm', content = InlineTemplate(self.getConfig()['configurations']['storm-env']['content']) @@ -111,18 +111,17 @@ class TestStormBase(RMFTestCase): owner = 'storm', ) return storm_yarn_content - + def call_storm_template_and_assert(self): import yaml_utils - storm_yarn_template = Template( - "storm.yaml.j2", - extra_imports=[yaml_utils.escape_yaml_propetry], - configurations = self.getConfig()['configurations']['storm-site']) - storm_yarn_content = storm_yarn_template.get_content() - - self.assertResourceCalled('File', '/etc/storm/conf/storm.yaml', - owner = 'storm', - content= storm_yarn_template, - group = 'hadoop' - ) - return storm_yarn_content + + with RMFTestCase.env as env: + storm_yarn_temlate = yaml_utils.yaml_config_template(self.getConfig()['configurations']['storm-site']) + + self.assertResourceCalled('File', '/etc/storm/conf/storm.yaml', + owner = 'storm', + content= storm_yarn_temlate, + group = 'hadoop' + ) + + return storm_yarn_temlate.get_content() http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json b/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json index 900eba4..dfcbe92 100644 --- a/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json +++ b/ambari-server/src/test/python/stacks/2.1/configs/default-storm-start.json @@ -76,6 +76,7 @@ "worker.heartbeat.frequency.secs": "1", "zmq.hwm": "0", "storm.zookeeper.connection.timeout": "15000", + "storm.thrift.transport": "{{storm_thrift_transport}}", "_storm.thrift.secure.transport": "SECURED_TRANSPORT_CLASS", "storm.messaging.netty.server_worker_threads": "1", "supervisor.worker.start.timeout.secs": "120", http://git-wip-us.apache.org/repos/asf/ambari/blob/008cc6ba/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json b/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json index 6b8f69a..206f18d 100644 --- a/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json +++ b/ambari-server/src/test/python/stacks/2.1/configs/secured-storm-start.json @@ -75,7 +75,8 @@ "topology.skip.missing.kryo.registrations": "false", "worker.heartbeat.frequency.secs": "1", "zmq.hwm": "0", - "storm.zookeeper.connection.timeout": "15000", + "storm.zookeeper.connection.timeout": "15000", + "storm.thrift.transport": "{{storm_thrift_transport}}", "_storm.thrift.secure.transport": "SECURED_TRANSPORT_CLASS", "storm.messaging.netty.server_worker_threads": "1", "supervisor.worker.start.timeout.secs": "120",