Repository: incubator-slider Updated Branches: refs/heads/feature/update_storm_for_93 8bfd66e81 -> eb6620e77
Temporary Storm package with Security Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eb6620e7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eb6620e7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eb6620e7 Branch: refs/heads/feature/update_storm_for_93 Commit: eb6620e77a1e8c97c5b65f8d098040d03594e478 Parents: 8bfd66e Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Thu Aug 28 18:21:18 2014 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Thu Aug 28 18:21:18 2014 -0700 ---------------------------------------------------------------------- app-packages/storm/README.txt | 4 +-- app-packages/storm/appConfig.json | 11 ++++---- app-packages/storm/configuration/storm-env.xml | 25 ++++++++++++++++++ app-packages/storm/metainfo.xml | 2 +- ...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 ------------ ...pache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE | 16 ++++++++++++ app-packages/storm/package/scripts/params.py | 11 +++++--- app-packages/storm/package/scripts/service.py | 8 ++++++ .../storm/package/templates/storm_jaas.conf.j2 | 27 +++++++++++++++++--- app-packages/storm/resources.json | 2 +- 10 files changed, 91 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/README.txt ---------------------------------------------------------------------- diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt index 971cf14..77474f1 100644 --- a/app-packages/storm/README.txt +++ b/app-packages/storm/README.txt @@ -26,10 +26,10 @@ Replace the placeholder tarball for Storm. rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/) - zip -r storm_v091.zip . + zip -r Apache_Storm_v_0_9_3.zip . Verify the content using - unzip -l "$@" storm_v091.zip + unzip -l "$@" Apache_Storm_v_0_9_3.zip While appConfig.json and resources.json are not required for the package they work well as the default configuration for Slider apps. So its advisable that when you http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json index b8454f9..d6dee84 100644 --- a/app-packages/storm/appConfig.json +++ b/app-packages/storm/appConfig.json @@ -3,12 +3,12 @@ "metadata": { }, "global": { - "application.def": "storm_v091.zip", + "application.def": "Apache_Storm_v_0_9_3.zip", "java_home": "/usr/jdk64/jdk1.7.0_45", "create.default.zookeeper.node": "true", "site.global.app_user": "yarn", - "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237", + "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100", "site.global.user_group": "hadoop", "site.global.security_enabled": "false", "site.global.ganglia_server_host": "${NN_HOST}", @@ -18,13 +18,14 @@ "site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}", "site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}", + "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}", "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']", "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}", "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm", "site.storm-site.transactional.zookeeper.root": "/transactional", "site.storm-site.storm.zookeeper.port": "2181", - "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", - "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", + "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", + "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper", "site.storm-site.drpc.invocations.port": "0", "site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}", @@ -32,7 +33,7 @@ "site.storm-site.nimbus.host": "${NIMBUS_HOST}", "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}", "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]", - "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", + "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", "site.storm-site.drpc.port": "0", "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}" }, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/configuration/storm-env.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm/configuration/storm-env.xml b/app-packages/storm/configuration/storm-env.xml index f315eb3..091c08d 100644 --- a/app-packages/storm/configuration/storm-env.xml +++ b/app-packages/storm/configuration/storm-env.xml @@ -22,6 +22,31 @@ <configuration> + <property> + <name>kerberos_domain</name> + <value></value> + <description>The kerberos domain to be used for this Storm cluster</description> + </property> + <property> + <name>storm_client_principal_name</name> + <value></value> + <description>The principal name for the Storm client to be used to communicate with Nimbus and Zookeeper</description> + </property> + <property> + <name>storm_server_principal_name</name> + <value></value> + <description>The principal name for the Storm server to be used by Nimbus</description> + </property> + <property> + <name>storm_client_keytab</name> + <value></value> + <description>The keytab file path for Storm client</description> + </property> + <property> + <name>storm_server_keytab</name> + <value></value> + <description>The keytab file path for Storm server</description> + </property> <!-- storm-env.sh --> <property> <name>content</name> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml index 70d369c..89ee01e 100644 --- a/app-packages/storm/metainfo.xml +++ b/app-packages/storm/metainfo.xml @@ -150,7 +150,7 @@ <packages> <package> <type>tarball</type> - <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name> + <name>files/apache-storm-0.9.3.0.2.5.0-100.tar.gz</name> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE deleted file mode 100644 index dd934d5..0000000 --- a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Replace with the actual storm package. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE new file mode 100644 index 0000000..dd934d5 --- /dev/null +++ b/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Replace with the actual storm package. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py index 3e1620b..93c6018 100644 --- a/app-packages/storm/package/scripts/params.py +++ b/app-packages/storm/package/scripts/params.py @@ -53,6 +53,11 @@ security_enabled = ( not is_empty(_authentication) and _authentication == 'kerbe if security_enabled: _hostname_lowercase = config['hostname'].lower() _kerberos_domain = config['configurations']['storm-env']['kerberos_domain'] - _storm_principal_name = config['configurations']['storm-env']['storm_principal_name'] - storm_jaas_principal = _storm_principal_name.replace('_HOST', _hostname_lowercase) - storm_keytab_path = config['configurations']['storm-env']['storm_keytab'] + _storm_client_principal_name = config['configurations']['storm-env']['storm_client_principal_name'] + _storm_server_principal_name = config['configurations']['storm-env']['storm_server_principal_name'] + + storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase) + storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab'] + storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase) + storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab'] + kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py index 13fcef2..dc3c733 100644 --- a/app-packages/storm/package/scripts/service.py +++ b/app-packages/storm/package/scripts/service.py @@ -52,6 +52,14 @@ def service( else: cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1") + if params.security_enabled: + if name == "nimbus": + Execute(format("{kinit_path_local} -kt {storm_server_keytab_path} {storm_user}"), + user=params.storm_user) + else: + Execute(format("{kinit_path_local} -kt {storm_client_keytab_path} {storm_user}"), + user=params.storm_user) + Execute(cmd, not_if=no_op_test, user=params.storm_user, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/package/templates/storm_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/templates/storm_jaas.conf.j2 b/app-packages/storm/package/templates/storm_jaas.conf.j2 index 4031d22..e68cd57 100644 --- a/app-packages/storm/package/templates/storm_jaas.conf.j2 +++ b/app-packages/storm/package/templates/storm_jaas.conf.j2 @@ -15,12 +15,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +StormServer { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{storm_server_keytab_path}}" + storeKey=true + useTicketCache=true + renewTGT=true + principal="{{storm_jaas_server_principal}}"; +}; +StormClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{storm_client_keytab_path}}" + storeKey=true + useTicketCache=true + renewTGT=true + serviceName="nimbus" + debug=true + principal="{{storm_jaas_client_principal}}"; +}; Client { com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true - keyTab="{{storm_keytab_path}}" + keyTab="{{storm_client_keytab_path}}" storeKey=true - useTicketCache=false + useTicketCache=true + renewTGT=true serviceName="zookeeper" - principal="{{storm_jaas_principal}}"; + principal="{{storm_jaas_client_principal}}"; }; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/resources.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json index b184a40..606dc39 100644 --- a/app-packages/storm/resources.json +++ b/app-packages/storm/resources.json @@ -13,7 +13,7 @@ }, "STORM_REST_API": { "yarn.role.priority": "2", - "yarn.component.instances": "1" + "yarn.component.instances": "0" }, "STORM_UI_SERVER": { "yarn.role.priority": "3",
