Repository: ambari Updated Branches: refs/heads/trunk fdaa98043 -> c33b8ee26
AMBARI-11734 - Missing Falcon conf files after rolling upgrade (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c33b8ee2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c33b8ee2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c33b8ee2 Branch: refs/heads/trunk Commit: c33b8ee26849accf516ca0afd0bb811513e65c9c Parents: fdaa980 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Fri Jun 5 15:44:34 2015 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Fri Jun 5 22:05:20 2015 -0400 ---------------------------------------------------------------------- .../libraries/functions/tar_archive.py | 26 ++++++++++++++++++++ .../package/scripts/falcon_server_upgrade.py | 10 +++----- .../1.4.0.2.0/package/scripts/flume_upgrade.py | 11 +++------ .../KNOX/0.5.0.2.2/package/scripts/upgrade.py | 12 ++++----- .../package/scripts/oozie_server_upgrade.py | 10 +++----- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 8 ++++++ .../stacks/2.1/FALCON/test_falcon_server.py | 8 ++++++ 7 files changed, 57 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py index efbf933..f1fb597 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py @@ -28,3 +28,29 @@ def archive_dir(output_filename, input_dir): tar.add(input_dir, arcname=os.path.basename(".")) finally: tar.close() + + +def archive_directory_dereference(archive, directory): + """ + Creates an archive of the specified directory. This will ensure that + symlinks are not included, but instead are followed for recursive inclusion. + :param archive: the name of the archive to create, including path + :param directory: the directory to include + :return: None + """ + tarball = None + try: + # !!! dereference must be TRUE since the conf is a symlink and we want + # its contents instead of the actual symlink + tarball = tarfile.open(archive, mode="w", dereference=True) + + # tar the files, chopping off everything in front of directory + # /foo/bar/conf/a, /foo/bar/conf/b, /foo/bar/conf/dir/c + # becomes + # a + # b + # dir/c + tarball.add(directory, arcname=os.path.relpath(directory, start=directory)) + finally: + if tarball: + tarball.close() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py index e6819d2..07cfbf6 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py @@ -23,6 +23,7 @@ import tempfile from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail +from resource_management.libraries.functions import tar_archive BACKUP_TEMP_DIR = "falcon-upgrade-backup" BACKUP_DATA_ARCHIVE = "falcon-local-backup.tar" @@ -51,13 +52,8 @@ def post_stop_backup(): if os.path.exists(archive): os.remove(archive) - tarball = None - try: - tarball = tarfile.open(archive, "w") - tarball.add(directory, arcname=os.path.basename(directory)) - finally: - if tarball: - tarball.close() + # backup the directory, following symlinks instead of including them + tar_archive.archive_directory_dereference(archive, directory) def pre_start_restore(): http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py index e2028fc..265bf5e 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py @@ -23,6 +23,7 @@ import tempfile from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail +from resource_management.libraries.functions import tar_archive BACKUP_TEMP_DIR = "flume-upgrade-backup" BACKUP_CONF_DIR_ARCHIVE = "flume-conf-backup.tar" @@ -50,13 +51,9 @@ def post_stop_backup(): if os.path.exists(archive): os.remove(archive) - tarball = None - try: - tarball = tarfile.open(archive, "w") - tarball.add(directory, arcname=os.path.basename(directory)) - finally: - if tarball: - tarball.close() + # backup the directory, following symlinks instead of including them + tar_archive.archive_directory_dereference(archive, directory) + def pre_start_restore(): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py index ee2b0e6..2c805fa 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py @@ -24,6 +24,7 @@ import tempfile from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail +from resource_management.libraries.functions import tar_archive BACKUP_TEMP_DIR = "knox-upgrade-backup" BACKUP_DATA_ARCHIVE = "knox-data-backup.tar" @@ -51,15 +52,12 @@ def backup_data(): if os.path.exists(archive): os.remove(archive) - tarball = None - try: - tarball = tarfile.open(archive, "w") - tarball.add(directory, arcname=os.path.basename(directory)) - finally: - if tarball: - tarball.close() + # backup the directory, following symlinks instead of including them + tar_archive.archive_directory_dereference(archive, directory) + return absolute_backup_dir + def _get_directory_mappings(): """ Gets a dictionary of directory to archive name that represents the http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py index 1afae6a..9d348e1 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py @@ -30,6 +30,7 @@ from resource_management.libraries.functions import Direction from resource_management.libraries.functions import format from resource_management.libraries.functions import compare_versions from resource_management.libraries.functions import format_hdp_stack_version +from resource_management.libraries.functions import tar_archive from resource_management.core.resources import File from resource_management.core.source import DownloadSource @@ -59,13 +60,8 @@ def backup_configuration(): if os.path.exists(archive): os.remove(archive) - tarball = None - try: - tarball = tarfile.open(archive, "w") - tarball.add(directory, arcname=os.path.basename(directory)) - finally: - if tarball: - tarball.close() + # backup the directory, following symlinks instead of including them + tar_archive.archive_directory_dereference(archive, directory) def restore_configuration(): http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index a75b7d0..ce0f52e 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -929,6 +929,14 @@ class TestOozieServer(RMFTestCase): self.assertTrue(tarfile_open_mock.called) self.assertEqual(tarfile_open_mock.call_count,2) + # check the call args for creation of the tarfile + call_args = tarfile_open_mock.call_args_list[0] + call_argument_tarfile = call_args[0][0] + call_kwargs = call_args[1] + + self.assertTrue("oozie-upgrade-backup/oozie-conf-backup.tar" in call_argument_tarfile) + self.assertEquals(True, call_kwargs["dereference"]) + self.assertTrue(chmod_mock.called) self.assertEqual(chmod_mock.call_count,1) chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511) http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py index 7c79bad..e7e0345 100644 --- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py +++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py @@ -212,6 +212,14 @@ class TestFalconServer(RMFTestCase): # 4 calls to tarfile.open (2 directories * read + write) self.assertTrue(tarfile_open_mock.called) self.assertEqual(tarfile_open_mock.call_count,4) + + # check the call args for creation of the tarfile + call_args = tarfile_open_mock.call_args_list[0] + call_argument_tarfile = call_args[0][0] + call_kwargs = call_args[1] + + self.assertTrue("falcon-upgrade-backup/falcon-conf-backup.tar" in call_argument_tarfile) + self.assertEquals(True, call_kwargs["dereference"]) @patch("resource_management.libraries.functions.security_commons.build_expectations") @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
