AMBARI-7856. Nimbus start fails on stage of deploying Ambari (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3c509fa5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3c509fa5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3c509fa5 Branch: refs/heads/branch-1.7.0 Commit: 3c509fa50f6466ac0e52f4713990d97a11e9e520 Parents: 4e439a8 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Mon Oct 20 14:47:36 2014 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Mon Oct 20 14:47:36 2014 +0300 ---------------------------------------------------------------------- .../services/STORM/package/scripts/params.py | 1 + .../services/STORM/package/scripts/service.py | 13 ++-- .../stacks/2.1/STORM/test_storm_drpc_server.py | 27 ++++---- .../stacks/2.1/STORM/test_storm_nimbus.py | 28 ++++---- .../2.1/STORM/test_storm_rest_api_service.py | 27 ++++---- .../stacks/2.1/STORM/test_storm_supervisor.py | 72 +++++++++----------- .../2.1/STORM/test_storm_supervisor_prod.py | 26 ++++--- .../stacks/2.1/STORM/test_storm_ui_server.py | 27 ++++---- 8 files changed, 103 insertions(+), 118 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3c509fa5/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py index 55b6dbc..89c389f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py @@ -42,6 +42,7 @@ conf_dir = "/etc/storm/conf" local_dir = config['configurations']['storm-site']['storm.local.dir'] user_group = config['configurations']['cluster-env']['user_group'] java64_home = config['hostLevelParams']['java_home'] +jps_binary = format("{java64_home}/bin/jps") nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port'] nimbus_host = config['configurations']['storm-site']['nimbus.host'] rest_api_port = "8745" http://git-wip-us.apache.org/repos/asf/ambari/blob/3c509fa5/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/service.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/service.py index 938dfc0..9dc42b9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/service.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/service.py @@ -38,16 +38,19 @@ def service( tries_count = 6 if name == 'ui': - process_cmd = "^java.+backtype.storm.ui.core$" + process_grep = "backtype.storm.ui.core$" elif name == "rest_api": - process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server") + process_grep = format("{rest_lib_dir}/storm-rest-.*\.jar$") else: - process_cmd = format("^java.+backtype.storm.daemon.{name}$") - - crt_pid_cmd = format("pgrep -f \"{process_cmd}\" && pgrep -f \"{process_cmd}\" > {pid_file}") + process_grep = format("storm.daemon.{name}$") + + find_proc = format("{jps_binary} -l | grep {process_grep}") + write_pid = format("{find_proc} | awk {{'print $1'}} > {pid_file}") + crt_pid_cmd = format("{find_proc} && {write_pid}") if action == "start": if name == "rest_api": + process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server") cmd = format("{process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log 2>&1") else: cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin storm {name} > {log_dir}/{name}.out 2>&1") http://git-wip-us.apache.org/repos/asf/ambari/blob/3c509fa5/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_drpc_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_drpc_server.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_drpc_server.py index 5d31bf0..189b874 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_drpc_server.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_drpc_server.py @@ -50,13 +50,12 @@ class TestStormDrpcServer(TestStormBase): path = ['/usr/bin'], user = 'storm', ) - - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.drpc$" && pgrep -f "^java.+backtype.storm.daemon.drpc$" > /var/run/storm/drpc.pid', - logoutput = True, - tries = 6, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.drpc$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.drpc$ | awk {'print $1'} > /var/run/storm/drpc.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 6, + user = 'storm', + try_sleep = 10, ) self.assertNoMoreResources() @@ -101,15 +100,13 @@ class TestStormDrpcServer(TestStormBase): path = ['/usr/bin'], user = 'storm', ) - - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.drpc$" && pgrep -f "^java.+backtype.storm.daemon.drpc$" > /var/run/storm/drpc.pid', - logoutput = True, - tries = 6, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.drpc$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.drpc$ | awk {'print $1'} > /var/run/storm/drpc.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 6, + user = 'storm', + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_secured(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/3c509fa5/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py index 59f8fca..9ab9730 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_nimbus.py @@ -50,15 +50,13 @@ class TestStormNimbus(TestStormBase): path = ['/usr/bin'], user = 'storm', ) - - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.nimbus$" && pgrep -f "^java.+backtype.storm.daemon.nimbus$" > /var/run/storm/nimbus.pid', - logoutput = True, - tries = 6, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.nimbus$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.nimbus$ | awk {'print $1'} > /var/run/storm/nimbus.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 6, + user = 'storm', + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_default(self): @@ -101,15 +99,13 @@ class TestStormNimbus(TestStormBase): path = ['/usr/bin'], user = 'storm', ) - - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.nimbus$" && pgrep -f "^java.+backtype.storm.daemon.nimbus$" > /var/run/storm/nimbus.pid', - logoutput = True, - tries = 6, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.nimbus$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.nimbus$ | awk {'print $1'} > /var/run/storm/nimbus.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 6, + user = 'storm', + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_secured(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/3c509fa5/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_rest_api_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_rest_api_service.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_rest_api_service.py index 7836f6b..b502859 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_rest_api_service.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_rest_api_service.py @@ -49,15 +49,13 @@ class TestStormRestApi(TestStormBase): path = ['/usr/bin'], user = 'storm', ) - - self.assertResourceCalled('Execute', 'pgrep -f "/usr/jdk64/jdk1.7.0_45/bin/java -jar /usr/lib/storm/contrib/storm-rest/`ls /usr/lib/storm/contrib/storm-rest | grep -wE storm-rest-[0-9.-]+\\.jar` server" && pgrep -f "/usr/jdk64/jdk1.7.0_45/bin/java -jar /usr/lib/storm/contrib/storm-rest/`ls /usr/lib/storm/contrib/storm-rest | grep -wE storm-rest-[0-9.-]+\\.jar` server" > /var/run/storm/restapi.pid', - logoutput = True, - tries = 6, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep /usr/lib/storm/contrib/storm-rest/storm-rest-.*\\.jar$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep /usr/lib/storm/contrib/storm-rest/storm-rest-.*\\.jar$ | awk {'print $1'} > /var/run/storm/restapi.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 6, + user = 'storm', + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_default(self): @@ -101,14 +99,13 @@ class TestStormRestApi(TestStormBase): user = 'storm', ) - self.assertResourceCalled('Execute', 'pgrep -f "/usr/jdk64/jdk1.7.0_45/bin/java -jar /usr/lib/storm/contrib/storm-rest/`ls /usr/lib/storm/contrib/storm-rest | grep -wE storm-rest-[0-9.-]+\\.jar` server" && pgrep -f "/usr/jdk64/jdk1.7.0_45/bin/java -jar /usr/lib/storm/contrib/storm-rest/`ls /usr/lib/storm/contrib/storm-rest | grep -wE storm-rest-[0-9.-]+\\.jar` server" > /var/run/storm/restapi.pid', - logoutput = True, - tries = 6, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep /usr/lib/storm/contrib/storm-rest/storm-rest-.*\\.jar$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep /usr/lib/storm/contrib/storm-rest/storm-rest-.*\\.jar$ | awk {'print $1'} > /var/run/storm/restapi.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 6, + user = 'storm', + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_secured(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/3c509fa5/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py index 33c64c7..1e0972f 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py @@ -49,29 +49,26 @@ class TestStormSupervisor(TestStormBase): path = ['/usr/bin'], user = 'storm', ) - - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.supervisor$" && pgrep -f "^java.+backtype.storm.daemon.supervisor$" > /var/run/storm/supervisor.pid', - logoutput = True, - tries = 6, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.supervisor$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.supervisor$ | awk {'print $1'} > /var/run/storm/supervisor.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 6, + user = 'storm', + try_sleep = 10, ) self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 PATH=$PATH:/usr/jdk64/jdk1.7.0_45/bin storm logviewer > /var/log/storm/logviewer.out 2>&1', - wait_for_finish = False, - not_if = 'ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1', - path = ['/usr/bin'], - user = 'storm' - ) - - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.logviewer$" && pgrep -f "^java.+backtype.storm.daemon.logviewer$" > /var/run/storm/logviewer.pid', - logoutput = True, - tries = 12, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10 + wait_for_finish = False, + path = ['/usr/bin'], + user = 'storm', + not_if = 'ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1', + ) + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.logviewer$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.logviewer$ | awk {'print $1'} > /var/run/storm/logviewer.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 12, + user = 'storm', + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_default(self): @@ -123,26 +120,25 @@ class TestStormSupervisor(TestStormBase): user = 'storm', ) - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.supervisor$" && pgrep -f "^java.+backtype.storm.daemon.supervisor$" > /var/run/storm/supervisor.pid', - logoutput = True, - tries = 6, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.supervisor$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.supervisor$ | awk {'print $1'} > /var/run/storm/supervisor.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 6, + user = 'storm', + try_sleep = 10, ) self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 PATH=$PATH:/usr/jdk64/jdk1.7.0_45/bin storm logviewer > /var/log/storm/logviewer.out 2>&1', - wait_for_finish = False, - not_if = 'ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1', - path = ['/usr/bin'], - user = 'storm' - ) - - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.logviewer$" && pgrep -f "^java.+backtype.storm.daemon.logviewer$" > /var/run/storm/logviewer.pid', - logoutput = True, - tries = 12, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10 + wait_for_finish = False, + path = ['/usr/bin'], + user = 'storm', + not_if = 'ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1', + ) + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.logviewer$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.logviewer$ | awk {'print $1'} > /var/run/storm/logviewer.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 12, + user = 'storm', + try_sleep = 10, ) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/3c509fa5/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py index 41c89c9..eb01146 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor_prod.py @@ -52,14 +52,13 @@ class TestStormSupervisor(TestStormBase): path = ['/usr/bin'], user = 'storm', ) - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.logviewer$" && pgrep -f "^java.+backtype.storm.daemon.logviewer$" > /var/run/storm/logviewer.pid', - logoutput = True, - tries = 12, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.logviewer$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.logviewer$ | awk {'print $1'} > /var/run/storm/logviewer.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 12, + user = 'storm', + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_default(self): @@ -110,14 +109,13 @@ class TestStormSupervisor(TestStormBase): path = ['/usr/bin'], user = 'storm', ) - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.logviewer$" && pgrep -f "^java.+backtype.storm.daemon.logviewer$" > /var/run/storm/logviewer.pid', - logoutput = True, - tries = 12, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.logviewer$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep storm.daemon.logviewer$ | awk {'print $1'} > /var/run/storm/logviewer.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 12, + user = 'storm', + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_secured(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/3c509fa5/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_ui_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_ui_server.py b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_ui_server.py index 256c376..365d926 100644 --- a/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_ui_server.py +++ b/ambari-server/src/test/python/stacks/2.1/STORM/test_storm_ui_server.py @@ -48,15 +48,13 @@ class TestStormUiServer(TestStormBase): path = ['/usr/bin'], user = 'storm', ) - - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.ui.core$" && pgrep -f "^java.+backtype.storm.ui.core$" > /var/run/storm/ui.pid', - logoutput = True, - tries = 6, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep backtype.storm.ui.core$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep backtype.storm.ui.core$ | awk {'print $1'} > /var/run/storm/ui.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 6, + user = 'storm', + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_default(self): @@ -100,14 +98,13 @@ class TestStormUiServer(TestStormBase): user = 'storm', ) - self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.ui.core$" && pgrep -f "^java.+backtype.storm.ui.core$" > /var/run/storm/ui.pid', - logoutput = True, - tries = 6, - user = 'storm', - path = ['/usr/bin'], - try_sleep = 10, + self.assertResourceCalled('Execute', "/usr/jdk64/jdk1.7.0_45/bin/jps -l | grep backtype.storm.ui.core$ && /usr/jdk64/jdk1.7.0_45/bin/jps -l | grep backtype.storm.ui.core$ | awk {'print $1'} > /var/run/storm/ui.pid", + logoutput = True, + path = ['/usr/bin'], + tries = 6, + user = 'storm', + try_sleep = 10, ) - self.assertNoMoreResources() def test_stop_secured(self):
