AMBARI-9228. Ambari Server setup to install and copy JCE policy file in-place (handle both Default / Custom JDK scenarios) (rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/049b6924 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/049b6924 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/049b6924 Branch: refs/heads/2.0-preview Commit: 049b692472151a4c3488afd2157dcd2ff509f265 Parents: 339e8a7 Author: Robert Levas <rle...@hortonworks.com> Authored: Wed Jan 21 15:32:27 2015 -0500 Committer: Yusaku Sako <yus...@hortonworks.com> Committed: Wed Jan 21 12:36:15 2015 -0800 ---------------------------------------------------------------------- ambari-server/sbin/ambari-server | 6 +- ambari-server/src/main/python/ambari-server.py | 195 +++++++++++++++++++ .../python/ambari_server/serverConfiguration.py | 2 + .../src/test/python/TestAmbariServer.py | 138 ++++++++++++- 4 files changed, 337 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/049b6924/ambari-server/sbin/ambari-server ---------------------------------------------------------------------- diff --git a/ambari-server/sbin/ambari-server b/ambari-server/sbin/ambari-server index c034f5d..168790b 100755 --- a/ambari-server/sbin/ambari-server +++ b/ambari-server/sbin/ambari-server @@ -111,6 +111,10 @@ case "$1" in echo -e "Setup ambari-server" $PYTHON /usr/sbin/ambari-server.py $@ ;; + setup-jce) + echo -e "Updating jce policy" + $PYTHON /usr/sbin/ambari-server.py $@ + ;; setup-ldap) echo -e "Setting up LDAP properties..." $PYTHON /usr/sbin/ambari-server.py $@ @@ -137,7 +141,7 @@ case "$1" in ;; *) echo "Usage: /usr/sbin/ambari-server - {start|stop|restart|setup|upgrade|status|upgradestack|setup-ldap|sync-ldap|setup-security|refresh-stack-hash|backup|restore} [options] + {start|stop|restart|setup|setup-jce|upgrade|status|upgradestack|setup-ldap|sync-ldap|setup-security|refresh-stack-hash|backup|restore} [options] Use usr/sbin/ambari-server <action> --help to get details on options available. Or, simply invoke ambari-server.py --help to print the options." exit 1 http://git-wip-us.apache.org/repos/asf/ambari/blob/049b6924/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index 7c08083..b75611f 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -81,6 +81,7 @@ UPGRADE_ACTION = "upgrade" UPGRADE_STACK_ACTION = "upgradestack" STATUS_ACTION = "status" SETUP_HTTPS_ACTION = "setup-https" +SETUP_JCE_ACTION = "setup-jce" LDAP_SETUP_ACTION = "setup-ldap" LDAP_SYNC_ACTION = "sync-ldap" SETUP_GANGLIA_HTTPS_ACTION = "setup-ganglia-https" @@ -403,6 +404,7 @@ DEFAULT_JDK16_LOCATION = "/usr/jdk64/jdk1.6.0_31" JDK_INDEX = 0 JDK_VERSION_REs = ["(jdk.*)/jre", "Creating (jdk.*)/jre"] CUSTOM_JDK_NUMBER = "3" +IS_CUSTOM_JDK = False JDK_MIN_FILESIZE = 5000 CREATE_JDK_DIR_CMD = "/bin/mkdir -p " + configDefaults.JDK_INSTALL_DIR MAKE_FILE_EXECUTABLE_CMD = "chmod a+x {0}" @@ -1560,6 +1562,7 @@ def install_jce_manualy(args): # def download_jdk(args): global JDK_INDEX + global IS_CUSTOM_JDK properties = get_ambari_properties() if properties == -1: err = "Error getting ambari properties" @@ -1604,6 +1607,7 @@ Enter choice (""" + jdk_num + "):", ) if jdk_num == CUSTOM_JDK_NUMBER: + IS_CUSTOM_JDK = True print_warning_msg("JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts.") print_warning_msg(jcePolicyWarn) args.java_home = get_validated_string_input("Path to JAVA_HOME: ", None, None, None, False, False) @@ -2025,6 +2029,174 @@ def verify_setup_allowed(): return 0 +def unpack_jce_policy(): + properties = get_ambari_properties() + jdk_path = properties.get_property(JAVA_HOME_PROPERTY) + jdk_security_path = jdk_path + os.sep + configDefaults.JDK_SECURITY_DIR + + jce_name = properties.get_property(JCE_NAME_PROPERTY) + jce_zip_path = configDefaults.SERVER_RESOURCES_DIR + os.sep + jce_name + unpack_cmd = 'unzip -o -j -q {0} -d {1}'.format(jce_zip_path, jdk_security_path) + + if os.path.exists(jdk_security_path) and os.path.exists(jce_zip_path) and validate_jdk(jdk_path): + try: + retcode, out, err = run_os_command(unpack_cmd) + if retcode != 0: + raise FatalException(retcode, err) + except Exception as e: + err = "Fail during the execution of '{0}'. {1}".format(unpack_cmd.format(jce_zip_path, jdk_security_path), e) + raise FatalException(1, err) + else: + err = "Can not execute {0}. The path {1}, {2} or {3} is invalid.".format(unpack_cmd, jdk_security_path, jce_zip_path, jdk_path) + raise FatalException(1, err) + +# +# Setup the JCE policy for Ambari Server. +# +def setup_jce_policy(path): + if os.path.exists(path): + copy_cmd = 'cp {0} {1}'.format(path, configDefaults.SERVER_RESOURCES_DIR) + try: + retcode, out, err = run_os_command(copy_cmd) + if retcode != 0: + raise FatalException(retcode, err) + except Exception as e: + err = "Fail during the execution of '{0}'. {1}".format(copy_cmd.format(path, configDefaults.SERVER_RESOURCES_DIR), e) + raise FatalException(1, err) + else: + err = "Can not run 'setup-jce'. Invalid path {0}.".format(path) + raise FatalException(1, err) + conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir()) + properties = get_ambari_properties() + zip_name = os.path.split(path)[1] + properties.process_pair(JCE_NAME_PROPERTY, zip_name) + try: + properties.store(open(conf_file, "w")) + except Exception, e: + print_error_msg('Could not write ambari config file "%s": %s' % (conf_file, e)) + + print 'Installing JCE policy...' + try: + unpack_jce_policy() + except FatalException as e: + err = 'Installing JCE failed: {0}. Exiting.'.format(e) + raise FatalException(e.code, err) + print 'NOTE: Restart Ambari Server to apply changes' + \ + ' ("ambari-server restart|stop|start")' + + +def unpack_jce_policy(): + properties = get_ambari_properties() + jdk_path = properties.get_property(JAVA_HOME_PROPERTY) + jdk_security_path = jdk_path + os.sep + configDefaults.JDK_SECURITY_DIR + + jce_name = properties.get_property(JCE_NAME_PROPERTY) + jce_zip_path = configDefaults.SERVER_RESOURCES_DIR + os.sep + jce_name + unpack_cmd = 'unzip -o -j -q {0} -d {1}'.format(jce_zip_path, jdk_security_path) + + if os.path.exists(jdk_security_path) and os.path.exists(jce_zip_path) and validate_jdk(jdk_path): + try: + retcode, out, err = run_os_command(unpack_cmd) + if retcode != 0: + raise FatalException(retcode, err) + except Exception as e: + err = "Fail during the execution of '{0}'. {1}".format(unpack_cmd.format(jce_zip_path, jdk_security_path), e) + raise FatalException(1, err) + else: + err = "Can not execute {0}. The path {1}, {2} or {3} is invalid.".format(unpack_cmd, jdk_security_path, jce_zip_path, jdk_path) + raise FatalException(1, err) + +# +# Setup the JCE policy for Ambari Server. +# +def setup_jce_policy(path): + if os.path.exists(path): + copy_cmd = 'cp {0} {1}'.format(path, configDefaults.SERVER_RESOURCES_DIR) + try: + retcode, out, err = run_os_command(copy_cmd) + if retcode != 0: + raise FatalException(retcode, err) + except Exception as e: + err = "Fail during the execution of '{0}'. {1}".format(copy_cmd.format(path, configDefaults.SERVER_RESOURCES_DIR), e) + raise FatalException(1, err) + else: + err = "Can not run 'setup-jce'. Invalid path {0}.".format(path) + raise FatalException(1, err) + conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir()) + properties = get_ambari_properties() + zip_name = os.path.split(path)[1] + properties.process_pair(JCE_NAME_PROPERTY, zip_name) + try: + properties.store(open(conf_file, "w")) + except Exception, e: + print_error_msg('Could not write ambari config file "%s": %s' % (conf_file, e)) + + print 'Installing JCE policy...' + try: + unpack_jce_policy() + except FatalException as e: + err = 'Installing JCE failed: {0}. Exiting.'.format(e) + raise FatalException(e.code, err) + print 'NOTE: Restart Ambari Server to apply changes' + \ + ' ("ambari-server restart|stop|start")' + + +def unpack_jce_policy(): + properties = get_ambari_properties() + jdk_path = properties.get_property(JAVA_HOME_PROPERTY) + jdk_security_path = jdk_path + os.sep + configDefaults.JDK_SECURITY_DIR + + jce_name = properties.get_property(JCE_NAME_PROPERTY) + jce_zip_path = configDefaults.SERVER_RESOURCES_DIR + os.sep + jce_name + unpack_cmd = 'unzip -o -j -q {0} -d {1}'.format(jce_zip_path, jdk_security_path) + + if os.path.exists(jdk_security_path) and os.path.exists(jce_zip_path) and validate_jdk(jdk_path): + try: + retcode, out, err = run_os_command(unpack_cmd) + if retcode != 0: + raise FatalException(retcode, err) + except Exception as e: + err = "Fail during the execution of '{0}'. {1}".format(unpack_cmd.format(jce_zip_path, jdk_security_path), e) + raise FatalException(1, err) + else: + err = "Can not execute {0}. The path {1}, {2} or {3} is invalid.".format(unpack_cmd, jdk_security_path, jce_zip_path, jdk_path) + raise FatalException(1, err) + +# +# Setup the JCE policy for Ambari Server. +# +def setup_jce_policy(path): + if os.path.exists(path): + copy_cmd = 'cp {0} {1}'.format(path, configDefaults.SERVER_RESOURCES_DIR) + try: + retcode, out, err = run_os_command(copy_cmd) + if retcode != 0: + raise FatalException(retcode, err) + except Exception as e: + err = "Fail during the execution of '{0}'. {1}".format(copy_cmd.format(path, configDefaults.SERVER_RESOURCES_DIR), e) + raise FatalException(1, err) + else: + err = "Can not run 'setup-jce'. Invalid path {0}.".format(path) + raise FatalException(1, err) + conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir()) + properties = get_ambari_properties() + zip_name = os.path.split(path)[1] + properties.process_pair(JCE_NAME_PROPERTY, zip_name) + try: + properties.store(open(conf_file, "w")) + except Exception, e: + print_error_msg('Could not write ambari config file "%s": %s' % (conf_file, e)) + + print 'Installing JCE policy...' + try: + unpack_jce_policy() + except FatalException as e: + err = 'Installing JCE failed: {0}. Exiting.'.format(e) + raise FatalException(e.code, err) + print 'NOTE: Restart Ambari Server to apply changes' + \ + ' ("ambari-server restart|stop|start")' + + # # Setup the Ambari Server. # @@ -2083,6 +2255,24 @@ def setup(args): err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e) raise FatalException(e.code, err) + print 'Installing JCE policy...' + try: + unpack_jce_policy() + except FatalException as e: + err = 'Installing JCE failed: {0}. Exiting.'.format(e) + raise FatalException(e.code, err) + + + + if not IS_CUSTOM_JDK: # If it's not a custom JDK, will also install JCE policy automatically + print 'Installing JCE policy...' + try: + unpack_jce_policy() + except FatalException as e: + err = 'Installing JCE failed: {0}. Exiting.'.format(e) + raise FatalException(e.code, err) + + print 'Completing setup...' retcode = configure_os_settings() if not retcode == 0: @@ -4223,6 +4413,8 @@ def main(): possible_args_numbers = [2,4] # OR elif action == BACKUP_ACTION or action == RESTORE_ACTION: possible_args_numbers = [1,2] + elif action == SETUP_JCE_ACTION: + possible_args_numbers = [2] else: possible_args_numbers = [1] @@ -4244,6 +4436,9 @@ def main(): start(options) elif action == STOP_ACTION: stop(options) + elif action == SETUP_JCE_ACTION: + path = args[1] + setup_jce_policy(path) elif action == RESET_ACTION: reset(options) elif action == STATUS_ACTION: http://git-wip-us.apache.org/repos/asf/ambari/blob/049b6924/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index 44c30ce..6964719 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -156,6 +156,8 @@ class ServerConfigDefaults(object): self.JDK_INSTALL_DIR = "" self.JDK_SEARCH_PATTERN = "" self.JAVA_EXE_SUBPATH = "" + self.JDK_SECURITY_DIR = "jre/lib/security" + self.SERVER_RESOURCES_DIR = "/var/lib/ambari-server/resources" # Configuration defaults self.DEFAULT_CONF_DIR = "" http://git-wip-us.apache.org/repos/asf/ambari/blob/049b6924/ambari-server/src/test/python/TestAmbariServer.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index 408c088..e0b390c 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -2418,6 +2418,119 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV result = _ambari_server_.find_jdk() self.assertEqual(result, "two") + @patch.object(_ambari_server_, "get_ambari_properties") + @patch("os.path.exists") + @patch.object(_ambari_server_, "run_os_command") + @patch.object(_ambari_server_, "validate_jdk") + def test_unpack_jce_policy(self, validate_jdk_mock, run_os_command_mock, exists_mock, get_ambari_properties_mock): + properties = MagicMock() + get_ambari_properties_mock.return_value = properties + exists_mock.return_value = True + run_os_command_mock.return_value = 0 , "", "" + validate_jdk_mock.return_value = True + + _ambari_server_.unpack_jce_policy() + self.assertTrue(run_os_command_mock.called) + self.assertTrue(validate_jdk_mock.called) + + # Testing with bad jdk_security_path or jce_zip_path + exists_mock.return_value = False + try: + _ambari_server_.unpack_jce_policy() + except FatalException: + self.assertTrue(True) + exists_mock.return_value = True + + # Testing with bad jdk path + validate_jdk_mock.return_value = False + try: + _ambari_server_.unpack_jce_policy() + except FatalException: + self.assertTrue(True) + validate_jdk_mock.return_value = True + + # Testing with return code distinct to 0 for run_os_command + run_os_command_mock.return_value = 3 , "", "" + try: + _ambari_server_.unpack_jce_policy() + except FatalException: + self.assertTrue(True) + run_os_command_mock.return_value = 0 , "", "" + + # Testing with an error produced by run_os_command + run_os_command_mock.reset_mock() + run_os_command_mock.side_effect = FatalException(1, "The command fails.") + try: + _ambari_server_.unpack_jce_policy() + except FatalException: + self.assertTrue(True) + + @patch("os.path.exists") + @patch.object(_ambari_server_, "run_os_command") + @patch("os.path.split") + @patch.object(_ambari_server_, "unpack_jce_policy") + @patch.object(_ambari_server_, "get_ambari_properties") + @patch.object(_ambari_server_, "search_file") + @patch("__builtin__.open") + def test_setup_jce_policy(self, open_mock, search_file_mock, get_ambari_properties_mock, unpack_jce_policy_mock, split_mock, run_os_command_mock, exists_mock): + exists_mock.return_value = True + run_os_command_mock.return_value = 0 , "", "" + properties = MagicMock() + unpack_jce_policy_mock.return_value = 0 + get_ambari_properties_mock.return_value = properties + conf_file = 'etc/ambari-server/conf/ambari.properties' + search_file_mock.return_value = conf_file + split_mock.return_value = [_ambari_server_.configDefaults.SERVER_RESOURCES_DIR, 'UnlimitedJCEPolicyJDK7.zip'] + + path = '/path/to/JCEPolicy.zip' + copy_cmd = 'cp {0} {1}'.format(path, _ambari_server_.configDefaults.SERVER_RESOURCES_DIR) + + _ambari_server_.setup_jce_policy(path) + run_os_command_mock.assert_called_with(copy_cmd) + self.assertTrue(unpack_jce_policy_mock.called) + self.assertTrue(get_ambari_properties_mock.called) + self.assertTrue(properties.store.called) + + # Testing with bad path + exists_mock.return_value = False + try: + _ambari_server_.setup_jce_policy(path) + except FatalException: + self.assertTrue(True) + exists_mock.return_value = True + + # Testing with return code distinct to 0 for run_os_command + run_os_command_mock.return_value = 2, "", "Fail" + try: + _ambari_server_.setup_jce_policy(path) + except FatalException: + self.assertTrue(True) + run_os_command_mock.return_value = 0, "", "" + + # Testing with an error produced by run_os_command + run_os_command_mock.reset_mock() + run_os_command_mock.side_effect = FatalException(1, "The command fails.") + try: + _ambari_server_.setup_jce_policy(path) + except FatalException: + self.assertTrue(True) + run_os_command_mock.return_value = 0, "", "" + + # Testing with an error produced by Properties.store function + properties.store.side_effect = Exception("Invalid file.") + try: + _ambari_server_.setup_jce_policy(path) + except Exception: + self.assertTrue(True) + properties.reset_mock() + + # Testing with an error produced by unpack_jce_policy + unpack_jce_policy_mock.side_effect = FatalException(1, "Can not install JCE policy") + try: + _ambari_server_.setup_jce_policy(path) + except FatalException: + self.assertTrue(True) + @patch("ambari_commons.firewall.run_os_command") @patch.object(OSCheck, "get_os_family") @patch.object(OSCheck, "get_os_type") @@ -2446,7 +2559,8 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV @patch.object(_ambari_server_, "extract_views") @patch.object(_ambari_server_, "adjust_directory_permissions") @patch.object(_ambari_server_, 'read_ambari_user') - def test_setup(self, read_ambari_user_mock, adjust_dirs_mock, extract_views_mock, proceedJDBCProperties_mock, is_server_runing_mock, is_root_mock, store_local_properties_mock, + @patch.object(_ambari_server_, "unpack_jce_policy") + def test_setup(self, unpack_jce_policy_mock, read_ambari_user_mock, adjust_dirs_mock, extract_views_mock, proceedJDBCProperties_mock, is_server_runing_mock, is_root_mock, store_local_properties_mock, is_local_database_mock, store_remote_properties_mock, setup_remote_db_mock, check_selinux_mock, check_jdbc_drivers_mock, check_ambari_user_mock, check_postgre_up_mock, setup_db_mock, configure_postgres_mock, @@ -2465,6 +2579,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV get_os_family_mock.return_value = OSConst.REDHAT_FAMILY run_os_command_mock.return_value = 3,"","" extract_views_mock.return_value = 0 + unpack_jce_policy_mock.return_value = 0 def reset_mocks(): is_jdbc_user_changed_mock.reset_mock() @@ -2574,6 +2689,19 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV self.assertFalse(check_selinux_mock.called) self.assertFalse(check_ambari_user_mock.called) + + # Test that unpack_jce_policy is called + reset_mocks() + _ambari_server_.setup(args) + self.assertTrue(unpack_jce_policy_mock.called) + + # Testing with an error produced by unpack_jce_policy + unpack_jce_policy_mock.side_effect = FatalException(1, "Can not install JCE policy") + try: + _ambari_server_.setup(args) + except FatalException: + self.assertTrue(True) + @patch.object(_ambari_server_, "get_remote_script_line") @patch.object(_ambari_server_, "is_server_runing") @patch.object(_ambari_server_, "get_YN_input") @@ -3958,7 +4086,8 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV @patch.object(_ambari_server_, "configure_os_settings") @patch('__builtin__.raw_input') @patch.object(_ambari_server_, "check_selinux") - def test_setup_remote_db_wo_client(self, check_selinux_mock, raw_input, configure_os_settings_mock, + @patch.object(_ambari_server_, "unpack_jce_policy") + def test_setup_remote_db_wo_client(self, unpack_jce_policy_mock, check_selinux_mock, raw_input, configure_os_settings_mock, download_jdk_mock, check_ambari_user_mock, is_root_mock, check_jdbc_drivers_mock, is_local_db_mock, store_remote_properties_mock, get_db_cli_tool_mock, get_YN_input, @@ -3982,6 +4111,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV download_jdk_mock.return_value = 0 configure_os_settings_mock.return_value = 0 verify_setup_allowed_method.return_value = 0 + unpack_jce_policy_mock.return_value = 0 try: _ambari_server_.setup(args) @@ -5407,7 +5537,8 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV @patch.object(_ambari_server_, "adjust_directory_permissions") @patch("sys.exit") @patch('__builtin__.raw_input') - def test_ambariServerSetupWithCustomDbName(self, raw_input, exit_mock, adjust_dirs_mock, extract_views_mock, store_password_file_mock, + @patch.object(_ambari_server_, "unpack_jce_policy") + def test_ambariServerSetupWithCustomDbName(self, unpack_jce_policy_mock, raw_input, exit_mock, adjust_dirs_mock, extract_views_mock, store_password_file_mock, get_is_secure_mock, setup_db_mock, is_root_mock, is_local_database_mock, check_selinux_mock, check_jdbc_drivers_mock, check_ambari_user_mock, check_postgre_up_mock, configure_postgres_mock, @@ -5438,6 +5569,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV get_os_type_mock.return_value = "" get_os_family_mock.return_value = OSConst.REDHAT_FAMILY run_os_command_mock.return_value = 3,"","" + unpack_jce_policy_mock.return_value = 0 new_db = "newDBName" args.dbms = "postgres"
