AMBARI-19038. Support migration of LDAP users & groups to PAM (rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f7fac037 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f7fac037 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f7fac037 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: f7fac03778fda337bc96ed49ed7507e1af118b7d Parents: f22256e Author: Robert Levas <rle...@hortonworks.com> Authored: Fri Jul 14 10:47:17 2017 -0400 Committer: Robert Levas <rle...@hortonworks.com> Committed: Fri Jul 14 10:47:17 2017 -0400 ---------------------------------------------------------------------- .../controllers/groups/GroupsEditCtrl.js | 3 + ambari-server/pom.xml | 2 +- ambari-server/sbin/ambari-server | 6 +- .../LdapToPamMigrationHelper.java | 73 ++++++++++++ .../server/security/authorization/Users.java | 4 + ambari-server/src/main/python/ambari-server.py | 14 ++- .../main/python/ambari_server/setupActions.py | 1 + .../main/python/ambari_server/setupSecurity.py | 119 ++++++++++++++++--- 8 files changed, 198 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f7fac037/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/groups/GroupsEditCtrl.js ---------------------------------------------------------------------- diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/groups/GroupsEditCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/groups/GroupsEditCtrl.js index 21d0fd6..a63ebe2 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/groups/GroupsEditCtrl.js +++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/groups/GroupsEditCtrl.js @@ -68,6 +68,7 @@ angular.module('ambariAdminConsole') function loadMembers(){ $scope.group.getMembers().then(function(members) { + $scope.group.groupTypeName = $t(GroupConstants.TYPES[$scope.group.group_type].LABEL_KEY); $scope.groupMembers = members; $scope.group.editingUsers = angular.copy($scope.groupMembers); }); @@ -81,6 +82,8 @@ angular.module('ambariAdminConsole') loadMembers(); }); + $scope.group.getGroupType(); + $scope.deleteGroup = function(group) { ConfirmationModal.show( $t('common.delete', { http://git-wip-us.apache.org/repos/asf/ambari/blob/f7fac037/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 878665e..70907da 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -1684,7 +1684,7 @@ <dependency> <groupId>net.java.dev.jna</groupId> <artifactId>jna</artifactId> - <version>4.1.0</version> + <version>4.3.0</version> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/ambari/blob/f7fac037/ambari-server/sbin/ambari-server ---------------------------------------------------------------------- diff --git a/ambari-server/sbin/ambari-server b/ambari-server/sbin/ambari-server index 24ec43a..1c6c612 100755 --- a/ambari-server/sbin/ambari-server +++ b/ambari-server/sbin/ambari-server @@ -137,6 +137,10 @@ case "${1:-}" in echo -e "Setting up PAM properties..." $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@ ;; + migrate-ldap-pam) + echo -e "Migration LDAP to PAM" + $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@ + ;; setup-ldap) echo -e "Setting up LDAP properties..." $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@ @@ -203,7 +207,7 @@ case "${1:-}" in ;; *) echo "Usage: $AMBARI_EXECUTABLE - {start|stop|reset|restart|upgrade|status|upgradestack|setup|setup-jce|setup-ldap|sync-ldap|set-current|setup-security|refresh-stack-hash|backup|restore|update-host-names|check-database|enable-stack|setup-sso|db-purge-history|install-mpack|uninstall-mpack|upgrade-mpack|setup-kerberos} [options] + {start|stop|reset|restart|upgrade|status|upgradestack|setup|setup-jce|setup-ldap|sync-ldap|set-current|setup-security|refresh-stack-hash|backup|restore|update-host-names|check-database|enable-stack|setup-sso|db-purge-history|install-mpack|uninstall-mpack|upgrade-mpack|setup-kerberos|setup-pam|migrate-ldap-pam} [options] Use $AMBARI_PYTHON_EXECUTABLE <action> --help to get details on options available. Or, simply invoke ambari-server.py --help to print the options." exit 1 http://git-wip-us.apache.org/repos/asf/ambari/blob/f7fac037/ambari-server/src/main/java/org/apache/ambari/server/security/authentication/LdapToPamMigrationHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/security/authentication/LdapToPamMigrationHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/security/authentication/LdapToPamMigrationHelper.java new file mode 100644 index 0000000..8a3a012 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/security/authentication/LdapToPamMigrationHelper.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.security.authentication; + +import java.sql.SQLException; + +import org.apache.ambari.server.audit.AuditLoggerModule; +import org.apache.ambari.server.controller.ControllerModule; +import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.orm.DBAccessor.DbType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; + +public class LdapToPamMigrationHelper { + private static final Logger LOG = LoggerFactory.getLogger(LdapToPamMigrationHelper.class); + + @Inject + private DBAccessor dbAccessor; + + /** + * Migrate LDAP user & groups to PAM + * + * @throws SQLException if an error occurs while executing the needed SQL statements + */ + private void migrateLdapUsersGroups() throws SQLException { + if (dbAccessor.getDbType() != DbType.ORACLE) { // Tested MYSQL, POSTGRES && MYSQL) + dbAccessor.executeQuery("UPDATE users SET user_type='PAM',ldap_user=0 WHERE ldap_user=1 and user_name not in (select user_name from (select user_name from users where user_type = 'PAM') as a)"); + dbAccessor.executeQuery("UPDATE groups SET group_type='PAM',ldap_group=0 WHERE ldap_group=1 and group_name not in (select group_name from (select group_name from groups where group_type = 'PAM') as a)"); + } else { // Tested ORACLE + dbAccessor.executeQuery("UPDATE users SET user_type='PAM',ldap_user=0 WHERE ldap_user=1 and user_name not in (select user_name from users where user_type = 'PAM')"); + dbAccessor.executeQuery("UPDATE groups SET group_type='PAM',ldap_group=0 WHERE ldap_group=1 and group_name not in (select group_name from groups where group_type = 'PAM')"); + } + } + + /** + * Support changes needed to migrate LDAP users & groups to PAM + * + * @param args Simple key value json map + */ + public static void main(String[] args) { + + try { + Injector injector = Guice.createInjector(new ControllerModule(), new AuditLoggerModule()); + LdapToPamMigrationHelper migrationHelper = injector.getInstance(LdapToPamMigrationHelper.class); + + migrationHelper.migrateLdapUsersGroups(); + + } catch (Throwable t) { + LOG.error("Caught exception on migration. Exiting...", t); + System.exit(1); + } + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f7fac037/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java b/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java index 9cdde8f..16c6c16 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java @@ -162,6 +162,10 @@ public class Users { if (userEntity != null) { userEntities.add(userEntity); } + userEntity = userDAO.findUserByNameAndType(userName, UserType.PAM); + if (userEntity != null) { + userEntities.add(userEntity); + } return (userEntities.isEmpty() || userEntities.size() > 1) ? null : new User(userEntities.get(0)); } http://git-wip-us.apache.org/repos/asf/ambari/blob/f7fac037/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index 5adcb04..8fcde77 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -52,8 +52,8 @@ from ambari_server.setupActions import BACKUP_ACTION, LDAP_SETUP_ACTION, LDAP_SY SETUP_ACTION, SETUP_SECURITY_ACTION,START_ACTION, STATUS_ACTION, STOP_ACTION, RESTART_ACTION, UPGRADE_ACTION, \ SETUP_JCE_ACTION, SET_CURRENT_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, \ SETUP_JCE_ACTION, SET_CURRENT_ACTION, ENABLE_STACK_ACTION, SETUP_SSO_ACTION, \ - DB_PURGE_ACTION, INSTALL_MPACK_ACTION, UNINSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION, PAM_SETUP_ACTION, KERBEROS_SETUP_ACTION -from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas, setup_pam + DB_PURGE_ACTION, INSTALL_MPACK_ACTION, UNINSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION, PAM_SETUP_ACTION, MIGRATE_LDAP_PAM_ACTION, KERBEROS_SETUP_ACTION +from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas, setup_pam, migrate_ldap_pam from ambari_server.userInput import get_validated_string_input from ambari_server.kerberos_setup import setup_kerberos @@ -540,6 +540,11 @@ def init_ldap_setup_parser_options(parser): parser.add_option('--ldap-sync-username-collisions-behavior', default=None, help="Handling behavior for username collisions [convert/skip] for LDAP sync", dest="ldap_sync_username_collisions_behavior") @OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) +def init_pam_setup_parser_options(parser): + parser.add_option('--pam-config-file', default=None, help="Path to the PAM configuration file", dest="pam_config_file") + parser.add_option('--pam-auto-create-groups', default=None, help="Automatically create groups for authenticated users [true/false]", dest="pam_auto_create_groups") + +@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) def init_set_current_parser_options(parser): parser.add_option('--cluster-name', default=None, help="Cluster name", dest="cluster_name") parser.add_option('--version-display-name', default=None, help="Display name of desired repo version", dest="desired_repo_version") @@ -783,7 +788,8 @@ def create_user_action_map(args, options): INSTALL_MPACK_ACTION: UserAction(install_mpack, options), UNINSTALL_MPACK_ACTION: UserAction(uninstall_mpack, options), UPGRADE_MPACK_ACTION: UserAction(upgrade_mpack, options), - PAM_SETUP_ACTION: UserAction(setup_pam), + PAM_SETUP_ACTION: UserAction(setup_pam, options), + MIGRATE_LDAP_PAM_ACTION: UserAction(migrate_ldap_pam, options), KERBEROS_SETUP_ACTION: UserAction(setup_kerberos, options) } return action_map @@ -814,7 +820,7 @@ def init_action_parser(action, parser): INSTALL_MPACK_ACTION: init_install_mpack_parser_options, UNINSTALL_MPACK_ACTION: init_uninstall_mpack_parser_options, UPGRADE_MPACK_ACTION: init_upgrade_mpack_parser_options, - PAM_SETUP_ACTION: init_empty_parser_options, + PAM_SETUP_ACTION: init_pam_setup_parser_options, KERBEROS_SETUP_ACTION: init_kerberos_setup_parser_options, } parser.add_option("-v", "--verbose", http://git-wip-us.apache.org/repos/asf/ambari/blob/f7fac037/ambari-server/src/main/python/ambari_server/setupActions.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupActions.py b/ambari-server/src/main/python/ambari_server/setupActions.py index 707cb84..61d20af 100644 --- a/ambari-server/src/main/python/ambari_server/setupActions.py +++ b/ambari-server/src/main/python/ambari_server/setupActions.py @@ -47,4 +47,5 @@ INSTALL_MPACK_ACTION = "install-mpack" UNINSTALL_MPACK_ACTION = "uninstall-mpack" UPGRADE_MPACK_ACTION = "upgrade-mpack" PAM_SETUP_ACTION = "setup-pam" +MIGRATE_LDAP_PAM_ACTION = "migrate-ldap-pam" KERBEROS_SETUP_ACTION = "setup-kerberos" http://git-wip-us.apache.org/repos/asf/ambari/blob/f7fac037/ambari-server/src/main/python/ambari_server/setupSecurity.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupSecurity.py b/ambari-server/src/main/python/ambari_server/setupSecurity.py index 17d1025..f175d7c 100644 --- a/ambari-server/src/main/python/ambari_server/setupSecurity.py +++ b/ambari-server/src/main/python/ambari_server/setupSecurity.py @@ -37,9 +37,9 @@ from ambari_commons.os_check import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons.os_utils import is_root, set_file_permissions, \ run_os_command, search_file, is_valid_filepath, change_owner, get_ambari_repo_file_full_name, get_file_owner -from ambari_server.serverConfiguration import configDefaults, \ +from ambari_server.serverConfiguration import configDefaults, parse_properties_file, \ encrypt_password, find_jdk, find_properties_file, get_alias_string, get_ambari_properties, get_conf_dir, \ - get_credential_store_location, get_is_persisted, get_is_secure, get_master_key_location, write_property, \ + get_credential_store_location, get_is_persisted, get_is_secure, get_master_key_location, get_db_type, write_property, \ get_original_master_key, get_value_from_properties, get_java_exe_path, is_alias_string, read_ambari_user, \ read_passwd_for_alias, remove_password_file, save_passwd_for_alias, store_password_file, update_properties_2, \ BLIND_PASSWORD, BOOTSTRAP_DIR_PROPERTY, IS_LDAP_CONFIGURED, JDBC_PASSWORD_FILENAME, JDBC_PASSWORD_PROPERTY, \ @@ -54,6 +54,8 @@ from ambari_server.serverUtils import is_server_runing, get_ambari_server_api_ba from ambari_server.setupActions import SETUP_ACTION, LDAP_SETUP_ACTION from ambari_server.userInput import get_validated_string_input, get_prompt_default, read_password, get_YN_input, quit_if_has_answer from ambari_server.serverClassPath import ServerClassPath +from ambari_server.dbConfiguration import DBMSConfigFactory, check_jdbc_drivers, \ + get_jdbc_driver_path, ensure_jdbc_driver_is_installed, LINUX_DBMS_KEYS_LIST logger = logging.getLogger(__name__) @@ -64,6 +66,9 @@ REGEX_TRUE_FALSE = "^(true|false)?$" REGEX_SKIP_CONVERT = "^(skip|convert)?$" REGEX_REFERRAL = "^(follow|ignore)?$" REGEX_ANYTHING = ".*" +LDAP_TO_PAM_MIGRATION_HELPER_CMD = "{0} -cp {1} " + \ + "org.apache.ambari.server.security.authentication.LdapToPamMigrationHelper" + \ + " >> " + configDefaults.SERVER_OUT_FILE + " 2>&1" CLIENT_SECURITY_KEY = "client.security" @@ -621,8 +626,12 @@ def setup_ldap(options): properties = get_ambari_properties() if get_value_from_properties(properties,CLIENT_SECURITY_KEY,"") == 'pam': - err = "PAM is configured. Can not setup LDAP." - raise FatalException(1, err) + query = "PAM is currently configured, do you wish to use LDAP instead [y/n] (n)? " + if get_YN_input(query, False): + pass + else: + err = "PAM is configured. Can not setup LDAP." + raise FatalException(1, err) isSecure = get_is_secure(properties) @@ -824,38 +833,112 @@ def ensure_can_start_under_current_user(ambari_user): return current_user class PamPropTemplate: - def __init__(self, properties, i_prop_name, i_prop_val_pattern, i_prompt_regex, i_allow_empty_prompt, i_prop_name_default=None): + def __init__(self, properties, i_option, i_prop_name, i_prop_val_pattern, i_prompt_regex, i_allow_empty_prompt, i_prop_name_default=None): self.prop_name = i_prop_name + self.option = i_option self.pam_prop_name = get_value_from_properties(properties, i_prop_name, i_prop_name_default) self.pam_prop_val_prompt = i_prop_val_pattern.format(get_prompt_default(self.pam_prop_name)) self.prompt_regex = i_prompt_regex self.allow_empty_prompt = i_allow_empty_prompt -def setup_pam(): +def init_pam_properties_list_reqd(properties, options): + properties = [ + PamPropTemplate(properties, options.pam_config_file, PAM_CONFIG_FILE, "PAM configuration file* {0}: ", REGEX_ANYTHING, False, "/etc/pam.d/ambari"), + PamPropTemplate(properties, options.pam_auto_create_groups, AUTO_GROUP_CREATION, "Do you want to allow automatic group creation* [true/false] {0}: ", REGEX_TRUE_FALSE, False, "false"), + ] + return properties + +def setup_pam(options): if not is_root(): - err = 'Ambari-server setup-pam should be run with ' \ - 'root-level privileges' + err = 'Ambari-server setup-pam should be run with root-level privileges' raise FatalException(4, err) properties = get_ambari_properties() if get_value_from_properties(properties,CLIENT_SECURITY_KEY,"") == 'ldap': - err = "LDAP is configured. Can not setup PAM." - raise FatalException(1, err) + query = "LDAP is currently configured, do you wish to use PAM instead [y/n] (n)? " + if get_YN_input(query, False): + pass + else: + err = "LDAP is configured. Can not setup PAM." + raise FatalException(1, err) + + pam_property_list_reqd = init_pam_properties_list_reqd(properties, options) pam_property_value_map = {} pam_property_value_map[CLIENT_SECURITY_KEY] = 'pam' - pamConfig = get_validated_string_input("Enter PAM configuration file: ", PAM_CONFIG_FILE, REGEX_ANYTHING, - "Invalid characters in the input!", False, False) - - pam_property_value_map[PAM_CONFIG_FILE] = pamConfig + for pam_prop in pam_property_list_reqd: + input = get_validated_string_input(pam_prop.pam_prop_val_prompt, pam_prop.pam_prop_name, pam_prop.prompt_regex, + "Invalid characters in the input!", False, pam_prop.allow_empty_prompt, + answer = pam_prop.option) + if input is not None and input != "": + pam_property_value_map[pam_prop.prop_name] = input - if get_YN_input("Do you want to allow automatic group creation [y/n] (y)? ", True): - pam_property_value_map[AUTO_GROUP_CREATION] = 'true' - else: - pam_property_value_map[AUTO_GROUP_CREATION] = 'false' + # Verify that the PAM config file exists, else show warning... + pam_config_file = pam_property_value_map[PAM_CONFIG_FILE] + if not os.path.exists(pam_config_file): + print_warning_msg("The PAM configuration file, {0} does not exist. " \ + "Please create it before restarting Ambari.".format(pam_config_file)) update_properties_2(properties, pam_property_value_map) print 'Saving...done' return 0 + +# +# Migration of LDAP users & groups to PAM +# +def migrate_ldap_pam(args): + properties = get_ambari_properties() + + if get_value_from_properties(properties,CLIENT_SECURITY_KEY,"") != 'pam': + err = "PAM is not configured. Please configure PAM authentication first." + raise FatalException(1, err) + + db_title = get_db_type(properties).title + confirm = get_YN_input("Ambari Server configured for %s. Confirm " + "you have made a backup of the Ambari Server database [y/n] (y)? " % db_title, True) + + if not confirm: + print_error_msg("Database backup is not confirmed") + return 1 + + jdk_path = get_java_exe_path() + if jdk_path is None: + print_error_msg("No JDK found, please run the \"setup\" " + "command to install a JDK automatically or install any " + "JDK manually to " + configDefaults.JDK_INSTALL_DIR) + return 1 + + # At this point, the args does not have the ambari database information. + # Augment the args with the correct ambari database information + parse_properties_file(args) + + ensure_jdbc_driver_is_installed(args, properties) + + print 'Migrating LDAP Users & Groups to PAM' + + serverClassPath = ServerClassPath(properties, args) + class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell() + + command = LDAP_TO_PAM_MIGRATION_HELPER_CMD.format(jdk_path, class_path) + + ambari_user = read_ambari_user() + current_user = ensure_can_start_under_current_user(ambari_user) + environ = generate_env(args, ambari_user, current_user) + + (retcode, stdout, stderr) = run_os_command(command, env=environ) + print_info_msg("Return code from LDAP to PAM migration command, retcode = " + str(retcode)) + if stdout: + print "Console output from LDAP to PAM migration command:" + print stdout + print + if stderr: + print "Error output from LDAP to PAM migration command:" + print stderr + print + if retcode > 0: + print_error_msg("Error executing LDAP to PAM migration, please check the server logs.") + else: + print_info_msg('LDAP to PAM migration completed') + return retcode
