Repository: ambari Updated Branches: refs/heads/trunk 3508a7ef6 -> 8bd43711a
AMBARI-14863. Oozie should update war after adding Falcon (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8bd43711 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8bd43711 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8bd43711 Branch: refs/heads/trunk Commit: 8bd43711a284b521af0a883ee5b7b77e3471e0ed Parents: 3508a7e Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Tue Mar 22 17:07:08 2016 +0200 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Tue Mar 22 17:07:08 2016 +0200 ---------------------------------------------------------------------- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 31 ++++++++++++++++++-- .../main/resources/scripts/Ambaripreupload.py | 28 ++++++++++++++++-- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 30 +++++++++++++++++++ 3 files changed, 84 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8bd43711/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index c503a38..5587380 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -26,6 +26,7 @@ from resource_management.core.source import InlineTemplate from resource_management.core.source import Template from resource_management.libraries.functions.format import format from resource_management.libraries.functions.version import compare_versions +from resource_management.libraries.functions.get_user_call_output import get_user_call_output from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.libraries.script.script import Script from resource_management.core.resources.packaging import Package @@ -199,12 +200,15 @@ def oozie_ownership(): def prepare_war(): """ - Attempt to call prepare-war command if the marker file doesn't exist or its content doesn't equal the expected command. - The marker file is stored in <stack-root>/current/oozie-server/.prepare_war_cmd + Attempt to call prepare-war command if the marker files don't exist or their content doesn't equal the expected. + The marker file for a command is stored in <stack-root>/current/oozie-server/.prepare_war_cmd + The marker file for a content of libext folder is stored in <stack-root>/current/oozie-server/.war_libext_content """ import params prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd") + libext_content_file = format("{oozie_home}/.war_libext_content") + list_libext_command = format("ls -l {oozie_libext_dir}") + " | awk '{print $9, $5}' | awk 'NF > 0'" # DON'T CHANGE THE VALUE SINCE IT'S USED TO DETERMINE WHETHER TO RUN THE COMMAND OR NOT BY READING THE MARKER FILE. # Oozie tmp dir should be /var/tmp/oozie and is already created by a function above. @@ -225,6 +229,23 @@ def prepare_war(): run_prepare_war = True Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} is missing.")) + return_code, libext_content, error_output = get_user_call_output(list_libext_command, user=params.oozie_user) + libext_content = libext_content.strip() + + if run_prepare_war == False: + if os.path.exists(libext_content_file): + old_content = "" + with open(libext_content_file, "r") as f: + old_content = f.read().strip() + + if libext_content != old_content: + run_prepare_war = True + Logger.info(format("Will run prepare war cmd since marker file {libext_content_file} has contents which differ.\n" \ + "Content of the folder {oozie_libext_dir} changed.")) + else: + run_prepare_war = True + Logger.info(format("Will run prepare war cmd since marker file {libext_content_file} is missing.")) + if run_prepare_war: # Time-consuming to run return_code, output = shell.call(command, user=params.oozie_user) @@ -236,11 +257,15 @@ def prepare_war(): Logger.error(message) raise Fail(message) - # Generate marker file + # Generate marker files File(prepare_war_cmd_file, content=command, mode=0644, ) + File(libext_content_file, + content=libext_content, + mode=0644, + ) else: Logger.info(format("No need to run prepare-war since marker file {prepare_war_cmd_file} already exists.")) http://git-wip-us.apache.org/repos/asf/ambari/blob/8bd43711/ambari-server/src/main/resources/scripts/Ambaripreupload.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py index 941539c..cbec3cf 100644 --- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py +++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py @@ -40,6 +40,7 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.get_user_call_output import get_user_call_output from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop from resource_management import Script @@ -312,6 +313,8 @@ with Environment() as env: # PREPARE-WAR [BEGIN] ############################################### prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd") + libext_content_file = format("{oozie_home}/.war_libext_content") + list_libext_command = format("ls -l {oozie_libext_dir}") + " | awk '{print $9, $5}' | awk 'NF > 0'" # DON'T CHANGE THE VALUE SINCE IT'S USED TO DETERMINE WHETHER TO RUN THE COMMAND OR NOT BY READING THE MARKER FILE. # Oozie tmp dir should be /var/tmp/oozie and is already created by a function above. @@ -332,6 +335,23 @@ with Environment() as env: run_prepare_war = True Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} is missing.")) + return_code, libext_content, error_output = get_user_call_output(list_libext_command, user=params.oozie_user) + libext_content = libext_content.strip() + + if run_prepare_war == False: + if os.path.exists(libext_content_file): + old_content = "" + with open(libext_content_file, "r") as f: + old_content = f.read().strip() + + if libext_content != old_content: + run_prepare_war = True + Logger.info(format("Will run prepare war cmd since marker file {libext_content_file} has contents which differ.\n" \ + "Content of the folder {oozie_libext_dir} changed.")) + else: + run_prepare_war = True + Logger.info(format("Will run prepare war cmd since marker file {libext_content_file} is missing.")) + if run_prepare_war: # Time-consuming to run return_code, output = shell.call(command, user=params.oozie_user) @@ -343,11 +363,15 @@ with Environment() as env: Logger.error(message) raise Fail(message) - # Generate marker file + # Generate marker files File(prepare_war_cmd_file, content=command_to_file, mode=0644, - ) + ) + File(libext_content_file, + content=libext_content, + mode=0644, + ) else: Logger.info(format("No need to run prepare-war since marker file {prepare_war_cmd_file} already exists.")) ############################################### http://git-wip-us.apache.org/repos/asf/ambari/blob/8bd43711/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index 407bb2f..ba61b3d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -40,6 +40,8 @@ class TestOozieServer(RMFTestCase): self.maxDiff = None @patch.object(shell, "call") + @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', ''))) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True])) def test_configure_default(self, call_mocks): call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", @@ -54,6 +56,8 @@ class TestOozieServer(RMFTestCase): self.assertNoMoreResources() @patch.object(shell, "call") + @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', ''))) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, False, True])) def test_configure_default_mysql(self, call_mocks): call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", @@ -261,6 +265,10 @@ class TestOozieServer(RMFTestCase): content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war', mode = 0644, ) + self.assertResourceCalled('File', '/usr/lib/oozie/.war_libext_content', + content = 'ext-2.2.zip', + mode = 0644, + ) self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', mode = 0644, ) @@ -271,6 +279,8 @@ class TestOozieServer(RMFTestCase): ) @patch.object(shell, "call") + @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', ''))) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, False, True])) def test_configure_existing_sqla(self, call_mocks): call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", @@ -482,6 +492,10 @@ class TestOozieServer(RMFTestCase): content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war', mode = 0644, ) + self.assertResourceCalled('File', '/usr/lib/oozie/.war_libext_content', + content = 'ext-2.2.zip', + mode = 0644, + ) self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', mode = 0644, ) @@ -494,6 +508,8 @@ class TestOozieServer(RMFTestCase): @patch.object(shell, "call") @patch("os.path.isfile") + @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', ''))) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True])) def test_start_default(self, isfile_mock, call_mocks): isfile_mock.return_value = True call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) @@ -572,6 +588,8 @@ class TestOozieServer(RMFTestCase): self.assertNoMoreResources() @patch.object(shell, "call") + @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', ''))) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True])) def test_configure_secured(self, call_mocks): call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", @@ -587,6 +605,8 @@ class TestOozieServer(RMFTestCase): @patch.object(shell, "call") @patch("os.path.isfile") + @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', ''))) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True])) def test_start_secured(self, isfile_mock, call_mocks): isfile_mock.return_value = True call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) @@ -848,6 +868,10 @@ class TestOozieServer(RMFTestCase): content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war', mode = 0644, ) + self.assertResourceCalled('File', '/usr/lib/oozie/.war_libext_content', + content = 'ext-2.2.zip', + mode = 0644, + ) self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', mode = 0644, ) @@ -1039,6 +1063,10 @@ class TestOozieServer(RMFTestCase): content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war -secure', mode = 0644, ) + self.assertResourceCalled('File', '/usr/lib/oozie/.war_libext_content', + content = 'ext-2.2.zip', + mode = 0644, + ) self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', mode = 0644, ) @@ -1049,6 +1077,8 @@ class TestOozieServer(RMFTestCase): ) @patch.object(shell, "call") + @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', ''))) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True])) def test_configure_default_hdp22(self, call_mocks): call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) config_file = "stacks/2.0.6/configs/default.json"