jaas fix for StormServer
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4b2f8aad Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4b2f8aad Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4b2f8aad Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4b2f8aad8d924cd1f2396b0057c885028210310b Parents: b2f2d79 8da1503 Author: Jon Maron <jma...@hortonworks.com> Authored: Mon Sep 8 09:27:56 2014 -0400 Committer: Jon Maron <jma...@hortonworks.com> Committed: Mon Sep 8 09:27:56 2014 -0400 ---------------------------------------------------------------------- README.md | 4 +- app-packages/accumulo/appConfig.json | 7 +- .../src/test/resources/appConfig_kerberos.json | 57 ++ .../test/resources/appConfig_monitor_ssl.json | 7 +- .../src/test/resources/appConfig_ssl.json | 7 +- app-packages/hbase/appConfig.json | 76 +-- app-packages/hbase/pom.xml | 2 +- app-packages/storm/README.txt | 2 +- app-packages/storm/appConfig.json | 13 +- app-packages/storm/metainfo.xml | 26 +- ...pache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE | 16 - app-packages/storm/package/scripts/params.py | 8 +- app-packages/storm/package/scripts/service.py | 43 +- .../storm/package/scripts/status_params.py | 4 +- .../storm/package/templates/config.yaml.j2 | 9 - app-packages/storm/pom.xml | 90 +++ app-packages/storm/resources.json | 12 +- app-packages/storm/src/assembly/storm.xml | 68 +++ pom.xml | 42 +- slider-agent/pom.xml | 4 +- .../src/main/python/agent/Controller.py | 1 + .../python/agent/CustomServiceOrchestrator.py | 10 +- slider-agent/src/main/python/agent/Register.py | 5 +- slider-agent/src/main/python/setup.py | 2 +- .../src/test/python/agent/TestRegistration.py | 5 +- slider-agent/src/test/python/python-wrap | 40 -- slider-agent/src/test/python/python-wrap.sh | 42 ++ slider-agent/src/test/python/unitTests.py | 36 +- slider-assembly/pom.xml | 23 +- slider-assembly/src/main/bash/README.md | 2 +- slider-assembly/src/main/bash/slider_destroy | 4 +- slider-assembly/src/main/scripts/slider | 2 + slider-assembly/src/main/scripts/slider.py | 12 +- slider-core/pom.xml | 4 +- .../org/apache/slider/api/InternalKeys.java | 11 + .../org/apache/slider/api/ResourceKeys.java | 13 - .../org/apache/slider/client/SliderClient.java | 43 +- .../common/params/AbstractActionArgs.java | 11 + .../slider/common/params/ActionFreezeArgs.java | 2 +- .../slider/common/params/ActionHelpArgs.java | 14 +- .../slider/common/params/ActionVersionArgs.java | 12 +- .../common/params/LaunchArgsAccessor.java | 2 +- .../slider/common/params/SliderActions.java | 8 +- .../apache/slider/common/tools/SliderUtils.java | 571 ++++++++++++++----- .../slider/core/build/InstanceBuilder.java | 5 +- .../apache/slider/core/zk/ZookeeperUtils.java | 3 +- .../slider/providers/agent/AgentKeys.java | 2 + .../providers/agent/AgentProviderService.java | 189 +++--- .../server/appmaster/RoleLaunchService.java | 8 +- .../server/appmaster/SliderAppMaster.java | 71 ++- .../server/appmaster/actions/ActionHalt.java | 6 +- .../appmaster/actions/ActionKillContainer.java | 20 +- .../appmaster/actions/ActionStartContainer.java | 8 +- .../appmaster/actions/ActionStopSlider.java | 6 +- .../server/appmaster/actions/AsyncAction.java | 47 +- .../actions/ProviderReportedContainerLoss.java | 13 +- .../actions/ProviderStartupCompleted.java | 4 - .../server/appmaster/actions/QueueAccess.java | 5 + .../server/appmaster/actions/QueueService.java | 11 +- .../actions/RegisterComponentInstance.java | 5 + .../actions/ReviewAndFlexApplicationSize.java | 43 ++ .../actions/UnregisterComponentInstance.java | 7 +- .../server/appmaster/monkey/ChaosEntry.java | 2 +- .../slider/server/appmaster/state/AppState.java | 11 +- .../server/appmaster/state/NodeEntry.java | 2 +- .../server/appmaster/state/RoleHistory.java | 6 +- .../appmaster/web/rest/agent/Register.java | 13 + .../services/utility/EndOfServiceWaiter.java | 84 +++ .../services/workflow/ForkedProcessService.java | 21 +- .../services/workflow/LongLivedProcess.java | 57 +- .../agent/AgentMiniClusterTestBase.groovy | 9 + .../agent/actions/TestActionExists.groovy | 2 +- .../agent/actions/TestActionVersion.groovy | 5 +- .../agent/freezethaw/TestFreezeCommands.groovy | 10 +- .../TestFreezeThawMasterlessAM.groovy | 8 +- .../freezethaw/TestFreezeUnknownCluster.groovy | 2 +- .../standalone/TestBuildStandaloneAM.groovy | 2 +- .../standalone/TestStandaloneAMDestroy.groovy | 6 +- .../standalone/TestStandaloneAMKill.groovy | 2 +- .../TestStandaloneAMMonkeyRestart.groovy | 8 +- .../standalone/TestStandaloneRegistryAM.groovy | 2 +- .../slider/client/TestCommonArgParsing.groovy | 2 +- .../tools/TestClientResourceRegistration.groovy | 3 + .../common/tools/TestConfigHelperHDFS.groovy | 3 + .../tools/TestExecutionEnvironment.groovy | 42 ++ .../common/tools/TestMiscSliderUtils.groovy | 3 + .../common/tools/TestWindowsSupport.groovy | 76 ++- .../common/tools/TestZKIntegration.groovy | 54 +- .../slider/providers/agent/AgentTestBase.groovy | 10 + .../agent/TestAgentAMManagementWS.groovy | 8 +- .../slider/providers/agent/TestAgentEcho.groovy | 6 +- .../curator/TestRegistryRestResources.groovy | 6 +- .../model/history/TestRoleHistoryRW.groovy | 4 +- .../publisher/TestPublisherRestResources.groovy | 26 +- .../apache/slider/test/SliderTestUtils.groovy | 11 +- .../slider/test/YarnMiniClusterTestBase.groovy | 32 +- .../test/YarnZKMiniClusterTestBase.groovy | 2 +- .../agent/TestAgentProviderService.java | 64 ++- .../services/workflow/EndOfServiceWaiter.java | 56 -- .../workflow/ProcessCommandFactory.java | 21 +- .../services/workflow/TestLongLivedProcess.java | 6 +- .../TestWorkflowForkedProcessService.java | 4 +- .../workflow/WorkflowServiceTestBase.java | 3 +- .../src/test/resources/example-slider-test.xml | 4 +- .../funtest/framework/CommandTestBase.groovy | 2 +- .../lifecycle/AgentClusterLifecycleIT.groovy | 8 +- .../src/test/manual/python/SliderTester.py | 4 +- .../providers/accumulo/AccumuloTestBase.groovy | 4 +- .../accumulo/live/TestAccFreezeThaw.groovy | 4 +- .../funtest/HBaseClusterLifecycleIT.groovy | 8 +- .../minicluster/HBaseMiniClusterTestBase.groovy | 1 - .../TestFreezeThawClusterFromArchive.groovy | 2 +- .../build/TestBuildThawClusterM1W1.groovy | 2 +- .../flexing/TestClusterFlex1To1.groovy | 2 +- ...reezeReconfigureThawLiveRegionService.groovy | 4 +- .../TestFreezeThawLiveRegionService.groovy | 4 +- .../minicluster/live/TestTwoLiveClusters.groovy | 4 +- src/test/clusters/sandbox/operations.md | 20 +- src/test/clusters/ubuntu-secure/operations.md | 14 +- 119 files changed, 1812 insertions(+), 794 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/README.txt ---------------------------------------------------------------------- diff --cc app-packages/storm/README.txt index 2bae3d2,49adfce..288604a --- a/app-packages/storm/README.txt +++ b/app-packages/storm/README.txt @@@ -17,19 -17,16 +17,19 @@@ How to create a Slider app package for Storm? -To create the app package you will need the Storm tarball and invoke mvn command -with appropriate parameters. +To create the app package you will need the Storm tarball copied to a specific location. +Various configurations provided in this sample are customized for apache-storm-0.9.1.2.1.1.0-237.tar.gz. +So if you use a different version you may need to edit a few config values. -Command: -mvn clean package -Pstorm-app-package -Dpkg.version=<version> - -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available> +Replace the placeholder tarball for Storm. + cp ~/Downloads/apache-storm-0.9.3.0.2.5.0-100.tar.gz package/files/ - rm package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE ++ rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE -Example: -mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578 - -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz -Dpkg.src=/Users/user1/Downloads +Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/) + zip -r Apache_Storm_v_0_9_3.zip . + +Verify the content using + unzip -l "$@" Apache_Storm_v_0_9_3.zip While appConfig.json and resources.json are not required for the package they work well as the default configuration for Slider apps. So its advisable that when you http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/appConfig.json ---------------------------------------------------------------------- diff --cc app-packages/storm/appConfig.json index d6dee84,1eae599..33e38ac --- a/app-packages/storm/appConfig.json +++ b/app-packages/storm/appConfig.json @@@ -3,14 -3,15 +3,15 @@@ "metadata": { }, "global": { - "application.def": "Apache_Storm_v_0_9_3.zip", + "application.def": "slider-storm-app-package-${pkg.version}.zip", "java_home": "/usr/jdk64/jdk1.7.0_45", "create.default.zookeeper.node": "true", + "system_configs": "core-site", "site.global.app_user": "yarn", - "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100", + "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}", "site.global.user_group": "hadoop", - + "site.global.security_enabled": "false", "site.global.ganglia_server_host": "${NN_HOST}", "site.global.ganglia_server_id": "Application2", "site.global.ganglia_enabled":"true", @@@ -33,7 -32,7 +32,7 @@@ "site.storm-site.nimbus.host": "${NIMBUS_HOST}", "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}", "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]", - "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", - "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", ++ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", "site.storm-site.drpc.port": "0", "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}" }, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/metainfo.xml ---------------------------------------------------------------------- diff --cc app-packages/storm/metainfo.xml index 89ee01e,bfd84e8..2c314b4 --- a/app-packages/storm/metainfo.xml +++ b/app-packages/storm/metainfo.xml @@@ -150,7 -130,7 +130,7 @@@ <packages> <package> <type>tarball</type> - <name>files/apache-storm-0.9.3.0.2.5.0-100.tar.gz</name> - <name>files/${pkg.name}</name> ++ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name> </package> </packages> </osSpecific> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/package/scripts/params.py ---------------------------------------------------------------------- diff --cc app-packages/storm/package/scripts/params.py index e0bf9e4,38e25fb..ff85765 --- a/app-packages/storm/package/scripts/params.py +++ b/app-packages/storm/package/scripts/params.py @@@ -57,7 -54,6 +53,7 @@@ if security_enabled storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase) storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab'] -- storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase) ++ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST',nimbus_host.lower()) + storm_jaas_stormclient_servicename = storm_jaas_server_principal.split("/")[0] storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/package/scripts/service.py ---------------------------------------------------------------------- diff --cc app-packages/storm/package/scripts/service.py index 0ec6413,3a5e795..2150cd4 --- a/app-packages/storm/package/scripts/service.py +++ b/app-packages/storm/package/scripts/service.py @@@ -47,10 -47,7 +47,7 @@@ def service crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk} > {pid_file}") if action == "start": - if name == "rest_api": - cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log") - else: - cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1") - cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1") ++ cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1") if params.security_enabled: if name == "nimbus": http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/resources.json ---------------------------------------------------------------------- diff --cc app-packages/storm/resources.json index 606dc39,017766c..f14b2c8 --- a/app-packages/storm/resources.json +++ b/app-packages/storm/resources.json @@@ -11,16 -11,12 +11,12 @@@ "yarn.role.priority": "1", "yarn.component.instances": "1" }, - "STORM_REST_API": { + "STORM_UI_SERVER": { "yarn.role.priority": "2", - "yarn.component.instances": "1" + "yarn.component.instances": "0" }, - "STORM_UI_SERVER": { - "yarn.role.priority": "3", - "yarn.component.instances": "1" - }, "DRPC_SERVER": { - "yarn.role.priority": "4", + "yarn.role.priority": "3", "yarn.component.instances": "1" }, "SUPERVISOR": { @@@ -28,4 -24,4 +24,4 @@@ "yarn.component.instances": "1" } } --} ++}
