SLIDER-181. Storm package updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/201686e2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/201686e2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/201686e2 Branch: refs/heads/feature/update_storm_for_93 Commit: 201686e2ea76d6f7f80554e5b9ad9471fce152be Parents: e7dd217 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Tue Sep 2 18:43:53 2014 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Tue Sep 2 18:43:53 2014 -0700 ---------------------------------------------------------------------- app-packages/storm/README.txt | 12 ++++----- app-packages/storm/appConfig.json | 15 +++++------ app-packages/storm/configuration/storm-env.xml | 25 ++++++++++++++++++ app-packages/storm/metainfo.xml | 26 +++---------------- ...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 ------------ ...pache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE | 16 ++++++++++++ app-packages/storm/package/scripts/params.py | 15 ++++++----- app-packages/storm/package/scripts/service.py | 8 ++++++ .../storm/package/scripts/status_params.py | 4 +-- .../storm/package/templates/config.yaml.j2 | 9 ------- .../storm/package/templates/storm_jaas.conf.j2 | 27 +++++++++++++++++--- app-packages/storm/resources.json | 10 +++----- 12 files changed, 101 insertions(+), 82 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/README.txt ---------------------------------------------------------------------- diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt index 971cf14..d4df85f 100644 --- a/app-packages/storm/README.txt +++ b/app-packages/storm/README.txt @@ -18,18 +18,18 @@ How to create a Slider app package for Storm? To create the app package you will need the Storm tarball copied to a specific location. -Various configurations provided in this sample are customized for apache-storm-0.9.1.2.1.1.0-237.tar.gz. +Various configurations provided in this sample are customized for apache-storm-0.9.3.2.2.0.0-578.tar.gz. So if you use a different version you may need to edit a few config values. Replace the placeholder tarball for Storm. - cp ~/Downloads/apache-storm-0.9.1.2.1.1.0-237.tar.gz package/files/ - rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE + cp ~/Downloads/apache-storm-0.9.3.2.2.0.0-578.tar.gz package/files/ + rm package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE -Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/) - zip -r storm_v091.zip . +Create a zip package at the root of the package (<slider enlistment>/app-packages/storm/) + zip -r apache-storm-0.9.3.zip . Verify the content using - unzip -l "$@" storm_v091.zip + unzip -l "$@" apache-storm-0.9.3.zip While appConfig.json and resources.json are not required for the package they work well as the default configuration for Slider apps. So its advisable that when you http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json index b8454f9..b5fe436 100644 --- a/app-packages/storm/appConfig.json +++ b/app-packages/storm/appConfig.json @@ -3,28 +3,27 @@ "metadata": { }, "global": { - "application.def": "storm_v091.zip", + "application.def": "apache-storm-0.9.3.zip", "java_home": "/usr/jdk64/jdk1.7.0_45", "create.default.zookeeper.node": "true", + "system_configs": "core-site", "site.global.app_user": "yarn", - "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237", + "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578", "site.global.user_group": "hadoop", - "site.global.security_enabled": "false", "site.global.ganglia_server_host": "${NN_HOST}", "site.global.ganglia_server_id": "Application2", "site.global.ganglia_enabled":"true", "site.global.ganglia_server_port": "8668", - "site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}", - "site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}", + "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}", "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']", "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}", "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm", "site.storm-site.transactional.zookeeper.root": "/transactional", "site.storm-site.storm.zookeeper.port": "2181", - "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", - "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", + "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", + "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper", "site.storm-site.drpc.invocations.port": "0", "site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}", @@ -32,7 +31,7 @@ "site.storm-site.nimbus.host": "${NIMBUS_HOST}", "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}", "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]", - "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", + "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", "site.storm-site.drpc.port": "0", "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}" }, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/configuration/storm-env.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm/configuration/storm-env.xml b/app-packages/storm/configuration/storm-env.xml index f315eb3..091c08d 100644 --- a/app-packages/storm/configuration/storm-env.xml +++ b/app-packages/storm/configuration/storm-env.xml @@ -22,6 +22,31 @@ <configuration> + <property> + <name>kerberos_domain</name> + <value></value> + <description>The kerberos domain to be used for this Storm cluster</description> + </property> + <property> + <name>storm_client_principal_name</name> + <value></value> + <description>The principal name for the Storm client to be used to communicate with Nimbus and Zookeeper</description> + </property> + <property> + <name>storm_server_principal_name</name> + <value></value> + <description>The principal name for the Storm server to be used by Nimbus</description> + </property> + <property> + <name>storm_client_keytab</name> + <value></value> + <description>The keytab file path for Storm client</description> + </property> + <property> + <name>storm_server_keytab</name> + <value></value> + <description>The keytab file path for Storm server</description> + </property> <!-- storm-env.sh --> <property> <name>content</name> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml index 70d369c..43535f4 100644 --- a/app-packages/storm/metainfo.xml +++ b/app-packages/storm/metainfo.xml @@ -21,7 +21,7 @@ <application> <name>STORM</name> <comment>Apache Hadoop Stream processing framework</comment> - <version>0.9.1.2.1</version> + <version>0.9.3</version> <exportedConfigs>storm-site</exportedConfigs> <exportGroups> @@ -29,10 +29,6 @@ <name>QuickLinks</name> <exports> <export> - <name>app.jmx</name> - <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value> - </export> - <export> <name>app.monitor</name> <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value> </export> @@ -55,7 +51,7 @@ <commandOrders> <commandOrder> <command>NIMBUS-START</command> - <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED + <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED </requires> </commandOrder> <commandOrder> @@ -67,10 +63,6 @@ <requires>NIMBUS-STARTED</requires> </commandOrder> <commandOrder> - <command>STORM_REST_API-START</command> - <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires> - </commandOrder> - <commandOrder> <command>STORM_UI_SERVER-START</command> <requires>NIMBUS-STARTED</requires> </commandOrder> @@ -91,18 +83,6 @@ </component> <component> - <name>STORM_REST_API</name> - <category>MASTER</category> - <autoStartOnFailure>true</autoStartOnFailure> - <appExports>QuickLinks-app.jmx</appExports> - <commandScript> - <script>scripts/rest_api.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - - <component> <name>SUPERVISOR</name> <category>SLAVE</category> <autoStartOnFailure>true</autoStartOnFailure> @@ -150,7 +130,7 @@ <packages> <package> <type>tarball</type> - <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name> + <name>files/apache-storm-0.9.3.2.2.0.0-578.tar.gz</name> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE deleted file mode 100644 index dd934d5..0000000 --- a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Replace with the actual storm package. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE new file mode 100644 index 0000000..dd934d5 --- /dev/null +++ b/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Replace with the actual storm package. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py index 3e1620b..a70bcb9 100644 --- a/app-packages/storm/package/scripts/params.py +++ b/app-packages/storm/package/scripts/params.py @@ -34,10 +34,6 @@ user_group = config['configurations']['global']['user_group'] java64_home = config['hostLevelParams']['java_home'] nimbus_host = config['configurations']['storm-site']['nimbus.host'] nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port'] -rest_api_port = config['configurations']['global']['rest_api_port'] -rest_api_admin_port = config['configurations']['global']['rest_api_admin_port'] -rest_api_conf_file = format("{conf_dir}/config.yaml") -rest_lib_dir = format("{app_root}/contrib/storm-rest") storm_bin = format("{app_root}/bin/storm") storm_env_sh_template = config['configurations']['storm-env']['content'] @@ -53,6 +49,11 @@ security_enabled = ( not is_empty(_authentication) and _authentication == 'kerbe if security_enabled: _hostname_lowercase = config['hostname'].lower() _kerberos_domain = config['configurations']['storm-env']['kerberos_domain'] - _storm_principal_name = config['configurations']['storm-env']['storm_principal_name'] - storm_jaas_principal = _storm_principal_name.replace('_HOST', _hostname_lowercase) - storm_keytab_path = config['configurations']['storm-env']['storm_keytab'] + _storm_client_principal_name = config['configurations']['storm-env']['storm_client_principal_name'] + _storm_server_principal_name = config['configurations']['storm-env']['storm_server_principal_name'] + + storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase) + storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab'] + storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase) + storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab'] + kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py index 13fcef2..dc3c733 100644 --- a/app-packages/storm/package/scripts/service.py +++ b/app-packages/storm/package/scripts/service.py @@ -52,6 +52,14 @@ def service( else: cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1") + if params.security_enabled: + if name == "nimbus": + Execute(format("{kinit_path_local} -kt {storm_server_keytab_path} {storm_user}"), + user=params.storm_user) + else: + Execute(format("{kinit_path_local} -kt {storm_client_keytab_path} {storm_user}"), + user=params.storm_user) + Execute(cmd, not_if=no_op_test, user=params.storm_user, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/scripts/status_params.py b/app-packages/storm/package/scripts/status_params.py index 5907446..2bf6870 100644 --- a/app-packages/storm/package/scripts/status_params.py +++ b/app-packages/storm/package/scripts/status_params.py @@ -28,10 +28,8 @@ pid_supervisor = format("{pid_dir}/supervisor.pid") pid_drpc = format("{pid_dir}/drpc.pid") pid_ui = format("{pid_dir}/ui.pid") pid_logviewer = format("{pid_dir}/logviewer.pid") -pid_rest_api = format("{pid_dir}/restapi.pid") pid_files = {"logviewer":pid_logviewer, "ui": pid_ui, "nimbus": pid_nimbus, "supervisor": pid_supervisor, - "drpc": pid_drpc, - "rest_api": pid_rest_api} \ No newline at end of file + "drpc": pid_drpc} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/templates/config.yaml.j2 ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/templates/config.yaml.j2 b/app-packages/storm/package/templates/config.yaml.j2 index 32d2c99..aa4ec46 100644 --- a/app-packages/storm/package/templates/config.yaml.j2 +++ b/app-packages/storm/package/templates/config.yaml.j2 @@ -16,15 +16,6 @@ nimbusHost: {{nimbus_host}} nimbusPort: {{nimbus_port}} -# HTTP-specific options. -http: - - # The port on which the HTTP server listens for service requests. - port: {{rest_api_port}} - - # The port on which the HTTP server listens for administrative requests. - adminPort: {{rest_api_admin_port}} - {% if ganglia_installed %} enableGanglia: {{ganglia_installed}} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/templates/storm_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/templates/storm_jaas.conf.j2 b/app-packages/storm/package/templates/storm_jaas.conf.j2 index 4031d22..e68cd57 100644 --- a/app-packages/storm/package/templates/storm_jaas.conf.j2 +++ b/app-packages/storm/package/templates/storm_jaas.conf.j2 @@ -15,12 +15,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +StormServer { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{storm_server_keytab_path}}" + storeKey=true + useTicketCache=true + renewTGT=true + principal="{{storm_jaas_server_principal}}"; +}; +StormClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{storm_client_keytab_path}}" + storeKey=true + useTicketCache=true + renewTGT=true + serviceName="nimbus" + debug=true + principal="{{storm_jaas_client_principal}}"; +}; Client { com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true - keyTab="{{storm_keytab_path}}" + keyTab="{{storm_client_keytab_path}}" storeKey=true - useTicketCache=false + useTicketCache=true + renewTGT=true serviceName="zookeeper" - principal="{{storm_jaas_principal}}"; + principal="{{storm_jaas_client_principal}}"; }; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/resources.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json index b184a40..017766c 100644 --- a/app-packages/storm/resources.json +++ b/app-packages/storm/resources.json @@ -11,20 +11,16 @@ "yarn.role.priority": "1", "yarn.component.instances": "1" }, - "STORM_REST_API": { - "yarn.role.priority": "2", - "yarn.component.instances": "1" - }, "STORM_UI_SERVER": { - "yarn.role.priority": "3", + "yarn.role.priority": "2", "yarn.component.instances": "1" }, "DRPC_SERVER": { - "yarn.role.priority": "4", + "yarn.role.priority": "3", "yarn.component.instances": "1" }, "SUPERVISOR": { - "yarn.role.priority": "5", + "yarn.role.priority": "4", "yarn.component.instances": "1" } }