Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-21450 56d2ade2b -> dd8d7109f
AMBARI-21595. Ambari Server Setup Code Fails Because of Missing Stack Upgrade (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dd8d7109 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dd8d7109 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dd8d7109 Branch: refs/heads/branch-feature-AMBARI-21450 Commit: dd8d7109f63f96cb363203be99047a31b415da33 Parents: 56d2ade Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Fri Jul 28 15:59:23 2017 +0300 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Fri Jul 28 15:59:23 2017 +0300 ---------------------------------------------------------------------- ambari-server/src/main/python/ambari-server.py | 9 +- .../main/python/ambari_server/setupActions.py | 1 - .../src/test/python/TestAmbariServer.py | 168 +------------------ .../src/test/python/TestServerUpgrade.py | 36 +--- 4 files changed, 12 insertions(+), 202 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/dd8d7109/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index b25cd16..60c6005 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -37,7 +37,7 @@ from ambari_server.dbConfiguration import DATABASE_NAMES, LINUX_DBMS_KEYS_LIST from ambari_server.serverConfiguration import configDefaults, get_ambari_properties, PID_NAME from ambari_server.serverUtils import is_server_runing, refresh_stack_hash, wait_for_server_to_stop from ambari_server.serverSetup import reset, setup, setup_jce_policy -from ambari_server.serverUpgrade import upgrade, upgrade_stack, set_current +from ambari_server.serverUpgrade import upgrade, set_current from ambari_server.setupHttps import setup_https, setup_truststore from ambari_server.setupMpacks import install_mpack, uninstall_mpack, upgrade_mpack, STACK_DEFINITIONS_RESOURCE_NAME, \ SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME @@ -49,9 +49,9 @@ from ambari_server.enableStack import enable_stack_version from ambari_server.setupActions import BACKUP_ACTION, LDAP_SETUP_ACTION, LDAP_SYNC_ACTION, PSTART_ACTION, \ REFRESH_STACK_HASH_ACTION, RESET_ACTION, RESTORE_ACTION, UPDATE_HOST_NAMES_ACTION, CHECK_DATABASE_ACTION, \ - SETUP_ACTION, SETUP_SECURITY_ACTION,START_ACTION, STATUS_ACTION, STOP_ACTION, RESTART_ACTION, UPGRADE_ACTION, \ - UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, SET_CURRENT_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, \ - UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, SET_CURRENT_ACTION, ENABLE_STACK_ACTION, SETUP_SSO_ACTION, \ + SETUP_ACTION, SETUP_SECURITY_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, RESTART_ACTION, UPGRADE_ACTION, \ + SETUP_JCE_ACTION, SET_CURRENT_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, \ + SETUP_JCE_ACTION, SET_CURRENT_ACTION, ENABLE_STACK_ACTION, SETUP_SSO_ACTION, \ DB_PURGE_ACTION, INSTALL_MPACK_ACTION, UNINSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION, PAM_SETUP_ACTION, MIGRATE_LDAP_PAM_ACTION, KERBEROS_SETUP_ACTION from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas, setup_pam, migrate_ldap_pam from ambari_server.userInput import get_validated_string_input @@ -753,7 +753,6 @@ def create_user_action_map(args, options): RESET_ACTION: UserAction(reset, options), STATUS_ACTION: UserAction(status, options), UPGRADE_ACTION: UserAction(upgrade, options), - UPGRADE_STACK_ACTION: UserActionPossibleArgs(upgrade_stack, [2, 4], args), LDAP_SETUP_ACTION: UserAction(setup_ldap, options), LDAP_SYNC_ACTION: UserAction(sync_ldap, options), SET_CURRENT_ACTION: UserAction(set_current, options), http://git-wip-us.apache.org/repos/asf/ambari/blob/dd8d7109/ambari-server/src/main/python/ambari_server/setupActions.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupActions.py b/ambari-server/src/main/python/ambari_server/setupActions.py index 926db98..61d20af 100644 --- a/ambari-server/src/main/python/ambari_server/setupActions.py +++ b/ambari-server/src/main/python/ambari_server/setupActions.py @@ -26,7 +26,6 @@ STOP_ACTION = "stop" RESTART_ACTION = "restart" RESET_ACTION = "reset" UPGRADE_ACTION = "upgrade" -UPGRADE_STACK_ACTION = "upgradestack" REFRESH_STACK_HASH_ACTION = "refresh-stack-hash" STATUS_ACTION = "status" SETUP_HTTPS_ACTION = "setup-https" http://git-wip-us.apache.org/repos/asf/ambari/blob/dd8d7109/ambari-server/src/test/python/TestAmbariServer.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index fb0bb70..5db7913 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -113,8 +113,8 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur from ambari_server.serverSetup import check_selinux, check_ambari_user, proceedJDBCProperties, SE_STATUS_DISABLED, SE_MODE_ENFORCING, configure_os_settings, \ download_and_install_jdk, prompt_db_properties, setup, \ AmbariUserChecks, AmbariUserChecksLinux, AmbariUserChecksWindows, JDKSetup, reset, setup_jce_policy, expand_jce_zip_file - from ambari_server.serverUpgrade import upgrade, upgrade_local_repo, change_objects_owner, upgrade_stack, \ - run_stack_upgrade, run_metainfo_upgrade, run_schema_upgrade, move_user_custom_actions, find_and_copy_custom_services + from ambari_server.serverUpgrade import upgrade, change_objects_owner, \ + run_schema_upgrade, move_user_custom_actions, find_and_copy_custom_services from ambari_server.setupHttps import is_valid_https_port, setup_https, import_cert_and_key_action, get_fqdn, \ generate_random_string, get_cert_info, COMMON_NAME_ATTR, is_valid_cert_exp, NOT_AFTER_ATTR, NOT_BEFORE_ATTR, \ SSL_DATE_FORMAT, import_cert_and_key, is_valid_cert_host, setup_truststore, \ @@ -4913,58 +4913,6 @@ class TestAmbariServer(TestCase): self.assertEqual(shutil_move_mock.call_count, 2) pass - - @patch("os.path.isdir", new = MagicMock(return_value=True)) - @patch("os.access", new = MagicMock(return_value=True)) - @patch("ambari_server.serverConfiguration.get_conf_dir") - @patch("ambari_server.serverUpgrade.run_os_command") - @patch("ambari_server.serverUpgrade.get_java_exe_path") - def test_run_stack_upgrade(self, java_exe_path_mock, run_os_command_mock, - get_conf_dir_mock): - java_exe_path_mock.return_value = "/usr/lib/java/bin/java" - run_os_command_mock.return_value = (0, None, None) - get_conf_dir_mock.return_value = '/etc/conf' - stackIdMap = {'HDP' : '2.0', 'repo_url' : 'http://test.com'} - - run_stack_upgrade(None, 'HDP', '2.0', 'http://test.com', None) - - self.assertTrue(java_exe_path_mock.called) - self.assertTrue(get_conf_dir_mock.called) - self.assertTrue(run_os_command_mock.called) - run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp \'/etc/conf:/usr/lib/ambari-server/*\' ' - 'org.apache.ambari.server.upgrade.StackUpgradeHelper ' - 'updateStackId ' + "'" + json.dumps(stackIdMap) + "'" + - ' > ' + os.sep + 'var' + os.sep + 'log' + os.sep + 'ambari-server' + os.sep + - 'ambari-server.out 2>&1') - pass - - @patch("os.path.isdir", new = MagicMock(return_value=True)) - @patch("os.access", new = MagicMock(return_value=True)) - @patch.object(ServerClassPath, "get_full_ambari_classpath_escaped_for_shell", - new = MagicMock(return_value = '/etc/conf' + os.pathsep + 'test' + os.pathsep + 'path12')) - @patch("ambari_server.serverConfiguration.get_conf_dir") - @patch("ambari_server.serverUpgrade.run_os_command") - @patch("ambari_server.serverUpgrade.get_java_exe_path") - def test_run_stack_upgrade_with_url_os(self, java_exe_path_mock, run_os_command_mock, - get_conf_dir_mock): - java_exe_path_mock.return_value = "/usr/lib/java/bin/java" - run_os_command_mock.return_value = (0, None, None) - get_conf_dir_mock.return_value = '/etc/conf' - stackIdMap = {'HDP' : '2.0', 'repo_url': 'http://test.com', 'repo_url_os': 'centos5,centos6'} - - run_stack_upgrade(None, 'HDP', '2.0', 'http://test.com', 'centos5,centos6') - - self.assertTrue(java_exe_path_mock.called) - self.assertTrue(get_conf_dir_mock.called) - self.assertTrue(run_os_command_mock.called) - run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp /etc/conf' + os.pathsep + 'test:path12 ' - 'org.apache.ambari.server.upgrade.StackUpgradeHelper ' - 'updateStackId ' + "'" + json.dumps(stackIdMap) + "'" + - ' > ' + os.sep + 'var' + os.sep + 'log' + os.sep + 'ambari-server' + os.sep + - 'ambari-server.out 2>&1') - pass - - @patch("os.path.isdir", new = MagicMock(return_value=True)) @patch("os.access", new = MagicMock(return_value=True)) @patch.object(ServerClassPath, "get_full_ambari_classpath_escaped_for_shell", @@ -5011,32 +4959,6 @@ class TestAmbariServer(TestCase): self.assertTrue(run_os_command_mock.called) run_os_command_mock.assert_called_with(command, env=environ) - @patch("os.path.isdir", new = MagicMock(return_value=True)) - @patch("os.access", new = MagicMock(return_value=True)) - @patch("ambari_server.serverConfiguration.get_conf_dir") - @patch.object(ServerClassPath, "get_full_ambari_classpath_escaped_for_shell", new = MagicMock(return_value = 'test' + os.pathsep + 'path12')) - @patch("ambari_server.serverUpgrade.run_os_command") - @patch("ambari_server.serverUpgrade.get_java_exe_path") - def test_run_metainfo_upgrade(self, java_exe_path_mock, run_os_command_mock, - get_conf_dir_mock): - java_exe_path_mock.return_value = "/usr/lib/java/bin/java" - run_os_command_mock.return_value = (0, None, None) - get_conf_dir_mock.return_value = '/etc/conf' - - json_map = {'a': 'http://newurl'} - run_metainfo_upgrade(None, json_map) - - self.assertTrue(java_exe_path_mock.called) - self.assertTrue(run_os_command_mock.called) - run_os_command_mock.assert_called_with('/usr/lib/java/bin/java ' - '-cp test' + os.pathsep + 'path12 ' - 'org.apache.ambari.server.upgrade.StackUpgradeHelper updateMetaInfo ' + - "'" + json.dumps(json_map) + "'" + - ' > ' + os.sep + 'var' + os.sep + 'log' + os.sep + 'ambari-server' + - os.sep + 'ambari-server.out 2>&1') - pass - - @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) @patch("os.path.isfile") @@ -5257,14 +5179,15 @@ class TestAmbariServer(TestCase): @patch("ambari_server.dbConfiguration.get_ambari_properties") @patch("ambari_server.serverConfiguration.get_ambari_properties") @patch("ambari_server.serverUpgrade.get_ambari_properties") - @patch("ambari_server.serverUpgrade.upgrade_local_repo") @patch("ambari_server.serverUpgrade.move_user_custom_actions") @patch("ambari_server.serverUpgrade.update_krb_jaas_login_properties") @patch("ambari_server.serverUpgrade.update_ambari_env") @patch("ambari_server.setupMpacks.get_replay_log_file") @patch("ambari_server.serverUpgrade.logger") @patch.object(PGConfig, "_change_db_files_owner", return_value=0) - def test_upgrade_from_161(self, change_db_files_owner_mock, logger_mock, get_replay_log_file_mock, update_ambari_env_mock, update_krb_jaas_login_properties_mock, move_user_custom_actions_mock, upgrade_local_repo_mock, get_ambari_properties_mock, + def test_upgrade_from_161(self, change_db_files_owner_mock, logger_mock, get_replay_log_file_mock, + update_ambari_env_mock, update_krb_jaas_login_properties_mock, + move_user_custom_actions_mock, get_ambari_properties_mock, get_ambari_properties_2_mock, get_ambari_properties_3_mock, get_ambari_version_mock, write_property_mock, is_root_mock, update_ambari_properties_mock, find_properties_file_mock, run_os_command_mock, run_schema_upgrade_mock, read_ambari_user_mock, print_warning_msg_mock, @@ -5479,11 +5402,10 @@ class TestAmbariServer(TestCase): @patch("ambari_server.dbConfiguration.get_ambari_properties") @patch("ambari_server.serverConfiguration.get_ambari_properties") @patch("ambari_server.serverUpgrade.get_ambari_properties") - @patch("ambari_server.serverUpgrade.upgrade_local_repo") @patch("ambari_server.serverUpgrade.move_user_custom_actions") @patch("ambari_server.serverUpgrade.update_krb_jaas_login_properties") @patch("ambari_server.serverUpgrade.update_ambari_env") - def test_upgrade(self, update_ambari_env_mock, update_krb_jaas_login_properties_mock, move_user_custom_actions, upgrade_local_repo_mock, + def test_upgrade(self, update_ambari_env_mock, update_krb_jaas_login_properties_mock, move_user_custom_actions, get_ambari_properties_mock, get_ambari_properties_2_mock, get_ambari_properties_3_mock, is_root_mock, get_ambari_version_mock, get_ambari_version_2_mock, parse_properties_file_mock, @@ -5549,7 +5471,6 @@ class TestAmbariServer(TestCase): self.assertTrue(print_warning_msg_mock.called) warning_args = print_warning_msg_mock.call_args[0][0] self.assertTrue("custom ambari user" in warning_args) - self.assertTrue(upgrade_local_repo_mock.called) self.assertTrue(move_user_custom_actions.called) args = reset_mocks() @@ -5664,10 +5585,9 @@ class TestAmbariServer(TestCase): @patch("ambari_server.dbConfiguration.get_ambari_properties") @patch("ambari_server.serverConfiguration.get_ambari_properties") @patch("ambari_server.serverUpgrade.get_ambari_properties") - @patch("ambari_server.serverUpgrade.upgrade_local_repo") @patch("ambari_server.serverUpgrade.move_user_custom_actions") @patch("ambari_server.serverUpgrade.update_krb_jaas_login_properties") - def test_upgrade(self, update_krb_jaas_login_properties_mock, move_user_custom_actions, upgrade_local_repo_mock, + def test_upgrade(self, update_krb_jaas_login_properties_mock, move_user_custom_actions, get_ambari_properties_mock, get_ambari_properties_2_mock, get_ambari_properties_3_mock, is_root_mock, get_ambari_version_mock, get_ambari_version_2_mock, parse_properties_file_mock, @@ -5734,7 +5654,6 @@ class TestAmbariServer(TestCase): self.assertTrue(print_warning_msg_mock.called) warning_args = print_warning_msg_mock.call_args[0][0] self.assertTrue("custom ambari user" in warning_args) - self.assertTrue(upgrade_local_repo_mock.called) self.assertTrue(move_user_custom_actions.called) args = reset_mocks() @@ -8313,79 +8232,6 @@ class TestAmbariServer(TestCase): validatorFunction = is_valid_filepath, answer='/kerberos/admin.keytab') pass - @patch("os.listdir") - @patch("os.path.exists") - @patch("ambari_server.serverUpgrade.load_stack_values") - @patch("ambari_server.serverUpgrade.get_ambari_properties") - @patch("ambari_server.serverUpgrade.run_metainfo_upgrade") - def test_upgrade_local_repo(self, - run_metainfo_upgrade_mock, - get_ambari_properties_mock, - load_stack_values_mock, - os_path_exists_mock, - os_listdir_mock): - - from mock.mock import call - args = MagicMock() - args.persistence_type = "local" - - def load_values_side_effect(*args, **kwargs): - res = {} - res['a'] = 'http://oldurl' - if -1 != args[1].find("HDPLocal"): - res['a'] = 'http://newurl' - return res - - load_stack_values_mock.side_effect = load_values_side_effect - - properties = Properties() - get_ambari_properties_mock.return_value = properties - os_path_exists_mock.return_value = 1 - os_listdir_mock.return_value = ['1.1'] - - upgrade_local_repo(args) - - self.assertTrue(get_ambari_properties_mock.called) - self.assertTrue(load_stack_values_mock.called) - self.assertTrue(run_metainfo_upgrade_mock.called) - run_metainfo_upgrade_mock.assert_called_with(args, {'a': 'http://newurl'}) - pass - - @patch("os.listdir") - @patch("os.path.exists") - @patch("ambari_server.serverUpgrade.load_stack_values") - @patch("ambari_server.serverUpgrade.get_ambari_properties") - @patch("ambari_server.serverUpgrade.run_metainfo_upgrade") - def test_upgrade_local_repo_nochange(self, - run_metainfo_upgrade_mock, - get_ambari_properties_mock, - load_stack_values_mock, - os_path_exists_mock, - os_listdir_mock): - - from mock.mock import call - args = MagicMock() - args.persistence_type = "local" - - def load_values_side_effect(*args, **kwargs): - res = {} - res['a'] = 'http://oldurl' - return res - - load_stack_values_mock.side_effect = load_values_side_effect - - properties = Properties() - get_ambari_properties_mock.return_value = properties - os_path_exists_mock.return_value = 1 - os_listdir_mock.return_value = ['1.1'] - - upgrade_local_repo(args) - - self.assertTrue(get_ambari_properties_mock.called) - self.assertTrue(load_stack_values_mock.called) - self.assertTrue(run_metainfo_upgrade_mock.called) - run_metainfo_upgrade_mock.assert_called_with(args, {}) - pass @patch("os.path.exists") @patch.object(ResourceFilesKeeper, "perform_housekeeping") http://git-wip-us.apache.org/repos/asf/ambari/blob/dd8d7109/ambari-server/src/test/python/TestServerUpgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestServerUpgrade.py b/ambari-server/src/test/python/TestServerUpgrade.py index 9235ff9..e61048a 100644 --- a/ambari-server/src/test/python/TestServerUpgrade.py +++ b/ambari-server/src/test/python/TestServerUpgrade.py @@ -36,7 +36,7 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur with patch("os.path.isdir", return_value = MagicMock(return_value=True)): with patch("os.access", return_value = MagicMock(return_value=True)): with patch.object(os_utils, "parse_log4j_file", return_value={'ambari.log.dir': '/var/log/ambari-server'}): - from ambari_server.serverUpgrade import set_current, SetCurrentVersionOptions, upgrade_stack + from ambari_server.serverUpgrade import set_current, SetCurrentVersionOptions import ambari_server os_utils.search_file = _search_file @@ -154,40 +154,6 @@ class TestServerUpgrade(TestCase): self.assertEquals(request.origin_req_host, '127.0.0.1') self.assertEquals(request.headers, {'X-requested-by': 'ambari', 'Authorization': 'Basic ZHVtbXlfc3RyaW5nOmR1bW15X3N0cmluZw=='}) - @patch("ambari_server.serverUpgrade.run_os_command") - @patch("ambari_server.serverUpgrade.get_java_exe_path") - @patch("ambari_server.serverConfiguration.get_ambari_properties") - @patch("ambari_server.serverUpgrade.get_ambari_properties") - @patch("ambari_server.serverUpgrade.check_database_name_property") - @patch("ambari_server.serverUpgrade.is_root") - def test_upgrade_stack(self, is_root_mock, c_d_n_p_mock, up_g_a_p_mock, server_g_a_p_mock, java_path_mock, run_os_mock): - - run_os_mock.return_value = 0, "", "" - - java_path_mock.return_value = "" - - is_root_mock.return_value = True - - def do_nothing(): - pass - c_d_n_p_mock.side_effect = do_nothing - - p = ambari_server.properties.Properties() - p._props = { - ambari_server.serverConfiguration.JDBC_DATABASE_PROPERTY: "mysql", - ambari_server.serverConfiguration.JDBC_DATABASE_NAME_PROPERTY: "ambari" - } - - up_g_a_p_mock.side_effect = [p, p] - server_g_a_p_mock.side_effect = [p] - - args = ["upgrade_stack", "HDP-2.3"] - upgrade_stack(args) - - self.assertTrue(run_os_mock.called) - command = run_os_mock.call_args_list[0][0][0] - self.assertTrue("StackUpgradeHelper" in command and "HDP" in command and "2.3" in command) - def testCurrentVersionOptions(self): # Negative test cases options = MagicMock()
