AMBARI-7417. Admin Views: Remove SYNC LDAP from the UI and create commandline ambari-server sync-ldap to call the API for syncing.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/21d784b6 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/21d784b6 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/21d784b6 Branch: refs/heads/trunk Commit: 21d784b68764899c8aa7262ba4b9a447911ddc22 Parents: d018aa6 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Fri Sep 19 11:21:04 2014 -0700 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Fri Sep 19 11:24:05 2014 -0700 ---------------------------------------------------------------------- .../main/resources/ui/admin-web/app/index.html | 1 - .../app/scripts/controllers/NavbarCtrl.js | 22 +- .../ui/admin-web/app/scripts/services/ldap.js | 60 - .../resources/ui/admin-web/app/styles/main.css | 10 +- .../ui/admin-web/app/views/leftNavbar.html | 9 - ambari-server/sbin/ambari-server | 4 + .../controller/AmbariManagementController.java | 10 +- .../AmbariManagementControllerImpl.java | 31 +- .../internal/ControllerResourceProvider.java | 53 +- .../server/security/authorization/Users.java | 17 +- .../security/ldap/AmbariLdapDataPopulator.java | 268 +++- ambari-server/src/main/python/ambari-server.py | 74 + .../ldap/AmbariLdapDataPopulatorTest.java | 1329 +++++++++++++++++- .../security/ldap/LdapPerformanceTest.java | 10 +- 14 files changed, 1673 insertions(+), 225 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-admin/src/main/resources/ui/admin-web/app/index.html ---------------------------------------------------------------------- diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/index.html b/ambari-admin/src/main/resources/ui/admin-web/app/index.html index 7ff0638..2fb9fce 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/index.html +++ b/ambari-admin/src/main/resources/ui/admin-web/app/index.html @@ -134,7 +134,6 @@ <script src="scripts/services/User.js"></script> <script src="scripts/services/Group.js"></script> <script src="scripts/services/View.js"></script> - <script src="scripts/services/ldap.js"></script> <script src="scripts/services/Cluster.js"></script> <script src="scripts/services/uiAlert.js"></script> <script src="scripts/services/PermissionLoader.js"></script> http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/NavbarCtrl.js ---------------------------------------------------------------------- diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/NavbarCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/NavbarCtrl.js index 9434cff..130d6fc 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/NavbarCtrl.js +++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/NavbarCtrl.js @@ -18,7 +18,7 @@ 'use strict'; angular.module('ambariAdminConsole') -.controller('NavbarCtrl',['$scope', 'Cluster', '$location', 'uiAlert', 'ROUTES', 'LDAP', 'ConfirmationModal', '$rootScope', function($scope, Cluster, $location, uiAlert, ROUTES, LDAP, ConfirmationModal, $rootScope) { +.controller('NavbarCtrl',['$scope', 'Cluster', '$location', 'uiAlert', 'ROUTES', 'ConfirmationModal', '$rootScope', function($scope, Cluster, $location, uiAlert, ROUTES, ConfirmationModal, $rootScope) { $scope.cluster = null; $scope.editCluster = { name : '', @@ -73,24 +73,4 @@ angular.module('ambariAdminConsole') var r = new RegExp( route.url.replace(/(:\w+)/, '\\w+')); return r.test($location.path()); }; - - $scope.isLDAPConfigured = false; - $scope.ldapData = {}; - LDAP.get().then(function(data) { - $scope.ldapData = data.data; - $scope.isLDAPConfigured = data.data['LDAP']['configured']; - }); - - $scope.syncLDAP = function() { - ConfirmationModal.show('Sync LDAP', 'Are you sure you want to sync LDAP?').then(function() { - LDAP.sync($scope.ldapData['LDAP'].groups, $scope.ldapData['LDAP'].users).then(function() { - uiAlert.success('LDAP synced successful'); - $rootScope.$evalAsync(function() { - $rootScope.LDAPSynced = true; - }); - }).catch(function(data) { - uiAlert.danger(data.data.status, data.data.message); - }); - }); - }; }]); http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/ldap.js ---------------------------------------------------------------------- diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/ldap.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/ldap.js deleted file mode 100644 index a911ec2..0000000 --- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/ldap.js +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -'use strict'; - -angular.module('ambariAdminConsole') -.factory('LDAP', ['$http', '$q', 'Settings', function($http, $q, Settings) { - - - return { - get: function() { - return $http({ - method: 'GET', - url: '/api/v1/controllers/ldap' - }); - }, - sync: function(groupsList, usersList) { - groupsList = Array.isArray(groupsList) ? groupsList : []; - usersList = Array.isArray(usersList) ? usersList : []; - return $http({ - method: 'PUT', - url: Settings.baseUrl + '/controllers/ldap', - data:[{ - LDAP:{ - "synced_groups": groupsList.join(','), - "synced_users": usersList.join(',') - } - }] - }); - }, - syncResource: function(resourceType, items) { - var items = items.map(function(item) { - var name = 'LDAP/synced_' + resourceType; - var obj = {}; - obj['LDAP/synced_' + resourceType] = item; - return obj; - }); - - return $http({ - method: 'POST', - url: '/api/v1/controllers/ldap', - data: items - }); - } - }; -}]); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css ---------------------------------------------------------------------- diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css b/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css index 4d6d081..131f452 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css +++ b/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css @@ -345,11 +345,6 @@ -o-transform: translateY(2px); transform: translateY(2px); } -.btn.disabled.syncldapbtn{ - pointer-events: auto; - background-color: #e6e6e6; - cursor: not-allowed; -} .btn.deleteuser-btn.disabled, .btn.deleteuser-btn[disabled], .btn.btn-delete-instance.disabled{ pointer-events: auto; cursor: not-allowed; @@ -466,9 +461,6 @@ font-size: 13px; color: #428bca; } -.left-navbar .panel-body #LDAP-button { - padding: 5px; -} .left-navbar .panel-body hr{ margin-top: 5px; margin-bottom: 5px; @@ -1099,4 +1091,4 @@ button.btn.btn-xs{ accordion .panel-group .panel{ overflow: visible; -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-admin/src/main/resources/ui/admin-web/app/views/leftNavbar.html ---------------------------------------------------------------------- diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/leftNavbar.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/leftNavbar.html index 8119428..4680e9a 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/views/leftNavbar.html +++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/leftNavbar.html @@ -85,15 +85,6 @@ <li ng-class="{active: isActive('users.list')}"><link-to route="users.list" class="userslist-link">Users</link-to></li> <li ng-class="{active: isActive('groups.list')}"><link-to route="groups.list" class="groupslist-link">Groups</link-to></li> </ul> - <hr> - <div id="LDAP-button" ng-switch="isLDAPConfigured"> - <a ng-switch-when="true" href class="btn btn-primary btn-block syncldapbtn" ng-click="syncLDAP()"> - <span class="glyphicon glyphicon-transfer pulldown2"></span> Sync LDAP - </a> - <a ng-switch-default href class="btn btn-default btn-block syncldapbtn disabled" tooltip="LDAP is not configured. To configure LDAP, run ambari-server setup-ldap from the command line."> - <span class="glyphicon glyphicon-transfer pulldown2"></span> Sync LDAP - </a> - </div> </div> </div> http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-server/sbin/ambari-server ---------------------------------------------------------------------- diff --git a/ambari-server/sbin/ambari-server b/ambari-server/sbin/ambari-server index f4b66eb..027bf87 100644 --- a/ambari-server/sbin/ambari-server +++ b/ambari-server/sbin/ambari-server @@ -115,6 +115,10 @@ case "$1" in echo -e "Setting up LDAP properties..." $PYTHON /usr/sbin/ambari-server.py $@ ;; + sync-ldap) + echo -e "Syncing with LDAP..." + $PYTHON /usr/sbin/ambari-server.py $@ + ;; setup-security) echo -e "Security setup options..." $PYTHON /usr/sbin/ambari-server.py $@ http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java index 13efd32..b932cbb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java @@ -24,6 +24,7 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.internal.RequestStageContainer; import org.apache.ambari.server.metadata.RoleCommandOrder; import org.apache.ambari.server.scheduler.ExecutionScheduleManager; +import org.apache.ambari.server.security.ldap.LdapBatchDto; import org.apache.ambari.server.security.ldap.LdapSyncDto; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; @@ -705,7 +706,14 @@ public interface AmbariManagementController { * @param groups groups to be synchronized * @throws AmbariException if synchronization data was invalid */ - public void synchronizeLdapUsersAndGroups(Set<String> users, Set<String> groups) throws AmbariException; + public LdapBatchDto synchronizeLdapUsersAndGroups(Set<String> users, Set<String> groups) throws AmbariException; + + /** + * Checks if LDAP sync process is running. + * + * @return true if LDAP sync is in progress + */ + public boolean isLdapSyncInProgress(); /** * Get configurations which are specific for a cluster (!not a service). http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 97137a2..8c0dfeb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -227,6 +227,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle final private String serverDB; final private String mysqljdbcUrl; + private boolean ldapSyncInProgress; + private Cache<ClusterRequest, ClusterResponse> clusterUpdateCache = CacheBuilder.newBuilder().expireAfterWrite(5, TimeUnit.MINUTES).build(); @@ -3675,9 +3677,30 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle } @Override - public synchronized void synchronizeLdapUsersAndGroups(Set<String> users, - Set<String> groups) throws AmbariException { - final LdapBatchDto batchInfo = ldapDataPopulator.synchronizeLdapUsersAndGroups(users, groups); - this.users.processLdapSync(batchInfo); + public boolean isLdapSyncInProgress() { + return ldapSyncInProgress; + } + + @Override + public synchronized LdapBatchDto synchronizeLdapUsersAndGroups(Set<String> users, Set<String> groups) + throws AmbariException { + ldapSyncInProgress = true; + try { + final LdapBatchDto batchInfo = new LdapBatchDto(); + if (users != null) { + ldapDataPopulator.synchronizeLdapUsers(users, batchInfo); + } else { + ldapDataPopulator.synchronizeAllLdapUsers(batchInfo); + } + if (groups != null) { + ldapDataPopulator.synchronizeLdapGroups(groups, batchInfo); + } else { + ldapDataPopulator.synchronizeAllLdapGroups(batchInfo); + } + this.users.processLdapSync(batchInfo); + return batchInfo; + } finally { + ldapSyncInProgress = false; + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ControllerResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ControllerResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ControllerResourceProvider.java index a3bd6d5..129824f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ControllerResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ControllerResourceProvider.java @@ -39,6 +39,7 @@ import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException; import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.security.ldap.LdapBatchDto; import org.apache.ambari.server.security.ldap.LdapGroupDto; import org.apache.ambari.server.security.ldap.LdapSyncDto; import org.apache.ambari.server.security.ldap.LdapUserDto; @@ -58,6 +59,8 @@ class ControllerResourceProvider extends AbstractControllerResourceProvider { protected static final String CONTROLLER_LDAP_SYNCED_USERS_PROPERTY_ID = PropertyHelper.getPropertyId("LDAP", "synced_users"); protected static final String CONTROLLER_LDAP_SYNCED_GROUPS_PROPERTY_ID = PropertyHelper.getPropertyId("LDAP", "synced_groups"); + protected static final String ALL_ENTRIES = "*"; + private static Set<String> pkPropertyIds = new HashSet<String>( Arrays.asList(new String[] { CONTROLLER_NAME_PROPERTY_ID })); @@ -190,39 +193,63 @@ class ControllerResourceProvider extends AbstractControllerResourceProvider { } // one request per each controller + Set<Resource> resources = new HashSet<Resource>(); for (final ControllerRequest controllerRequest: requests) { - modifyResources(new Command<Void>() { + Resource resource = modifyResources(new Command<Resource>() { @Override - public Void invoke() throws AmbariException { + public Resource invoke() throws AmbariException { + Resource resource = null; switch (ControllerType.getByName(controllerRequest.getName())) { case LDAP: - Set<String> users = new HashSet<String>(); + resource = new ResourceImpl(Resource.Type.Controller); + Set<String> users = null; if (controllerRequest.getPropertyMap().containsKey(CONTROLLER_LDAP_SYNCED_USERS_PROPERTY_ID)) { final String userCsv = (String) controllerRequest.getPropertyMap().get(CONTROLLER_LDAP_SYNCED_USERS_PROPERTY_ID); - for (String user: userCsv.split(",")) { - if (StringUtils.isNotEmpty(user)) { - users.add(user.toLowerCase()); + if (!userCsv.trim().equals(ALL_ENTRIES)) { + users = new HashSet<String>(); + for (String user: userCsv.split(",")) { + if (StringUtils.isNotEmpty(user)) { + users.add(user.toLowerCase()); + } } } } - Set<String> groups = new HashSet<String>(); + Set<String> groups = null; if (controllerRequest.getPropertyMap().containsKey(CONTROLLER_LDAP_SYNCED_GROUPS_PROPERTY_ID)) { final String groupCsv = (String) controllerRequest.getPropertyMap().get(CONTROLLER_LDAP_SYNCED_GROUPS_PROPERTY_ID); - for (String group: groupCsv.split(",")) { - if (StringUtils.isNotEmpty(group)) { - groups.add(group.toLowerCase()); + if (!groupCsv.trim().equals(ALL_ENTRIES)) { + groups = new HashSet<String>(); + for (String group : groupCsv.split(",")) { + if (StringUtils.isNotEmpty(group)) { + groups.add(group.toLowerCase()); + } } } } - getManagementController().synchronizeLdapUsersAndGroups(users, groups); + if (!getManagementController().isLdapSyncInProgress()) { + LdapBatchDto syncInfo = getManagementController().synchronizeLdapUsersAndGroups(users, groups); + resource.setProperty("Sync/status", "successful"); + resource.setProperty("Sync/summary/Users/created", syncInfo.getUsersToBeCreated().size()); + resource.setProperty("Sync/summary/Users/updated", syncInfo.getUsersToBecomeLdap().size()); + resource.setProperty("Sync/summary/Users/removed", syncInfo.getUsersToBeRemoved().size()); + resource.setProperty("Sync/summary/Groups/created", syncInfo.getGroupsToBeCreated().size()); + resource.setProperty("Sync/summary/Groups/updated", syncInfo.getGroupsToBecomeLdap().size()); + resource.setProperty("Sync/summary/Groups/removed", syncInfo.getGroupsToBeRemoved().size()); + resource.setProperty("Sync/summary/Memberships/created", syncInfo.getMembershipToAdd().size()); + resource.setProperty("Sync/summary/Memberships/removed", syncInfo.getMembershipToRemove().size()); + } else { + resource.setProperty("Sync/status", "not started"); + resource.setProperty("Sync/summary", "Another sync is already running"); + } break; } - return null; + return resource; } }); + resources.add(resource); } - return getRequestStatus(null); + return getRequestStatus(null, resources); } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java b/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java index 1cd5b95..fd2f77b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java @@ -638,13 +638,6 @@ public class Users { userDAO.create(usersToCreate); groupDAO.create(groupsToCreate); - // remove membership - final Set<MemberEntity> membersToRemove = new HashSet<MemberEntity>(); - for (LdapUserGroupMemberDto member: batchInfo.getMembershipToRemove()) { - membersToRemove.add(memberDAO.findByUserAndGroup(member.getUserName(), member.getGroupName())); - } - memberDAO.remove(membersToRemove); - // create membership final Set<MemberEntity> membersToCreate = new HashSet<MemberEntity>(); final Set<GroupEntity> groupsToUpdate = new HashSet<GroupEntity>(); @@ -660,6 +653,16 @@ public class Users { memberDAO.create(membersToCreate); groupDAO.merge(groupsToUpdate); // needed for Derby DB as it doesn't fetch newly added members automatically + // remove membership + final Set<MemberEntity> membersToRemove = new HashSet<MemberEntity>(); + for (LdapUserGroupMemberDto member: batchInfo.getMembershipToRemove()) { + MemberEntity memberEntity = memberDAO.findByUserAndGroup(member.getUserName(), member.getGroupName()); + if (memberEntity != null) { + membersToRemove.add(memberEntity); + } + } + memberDAO.remove(membersToRemove); + // clear cached entities entityManagerProvider.get().getEntityManagerFactory().getCache().evictAll(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java b/ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java index 2342bd4..f07c6d0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java @@ -42,7 +42,11 @@ import org.springframework.ldap.core.ContextMapper; import org.springframework.ldap.core.DirContextAdapter; import org.springframework.ldap.core.LdapTemplate; import org.springframework.ldap.core.support.LdapContextSource; +import org.springframework.ldap.filter.AndFilter; import org.springframework.ldap.filter.EqualsFilter; +import org.springframework.ldap.filter.Filter; +import org.springframework.ldap.filter.LikeFilter; +import org.springframework.ldap.filter.OrFilter; import org.springframework.security.core.userdetails.UsernameNotFoundException; import com.google.inject.Inject; @@ -80,6 +84,7 @@ public class AmbariLdapDataPopulator { public AmbariLdapDataPopulator(Configuration configuration, Users users) { this.configuration = configuration; this.users = users; + this.ldapServerProperties = configuration.getLdapServerProperties(); } /** @@ -143,69 +148,136 @@ public class AmbariLdapDataPopulator { } /** - * Performs synchronization of given sets of usernames and groupnames. + * Performs synchronization of all groups. * - * @param users set of users to synchronize - * @param groups set of groups to synchronize * @throws AmbariException if synchronization failed for any reason */ - public LdapBatchDto synchronizeLdapUsersAndGroups(Set<String> users, - Set<String> groups) throws AmbariException { - final LdapBatchDto batchInfo = new LdapBatchDto(); + public LdapBatchDto synchronizeAllLdapGroups(LdapBatchDto batchInfo) throws AmbariException { - // validate request - final Set<LdapUserDto> externalUsers = getExternalLdapUserInfo(); - final Map<String, LdapUserDto> externalUsersMap = new HashMap<String, LdapUserDto>(); - for (LdapUserDto user: externalUsers) { - externalUsersMap.put(user.getUserName(), user); + Set<LdapGroupDto> externalLdapGroupInfo = getExternalLdapGroupInfo(); + + final Map<String, Group> internalGroupsMap = getInternalGroups(); + final Map<String, User> internalUsersMap = getInternalUsers(); + + for (LdapGroupDto groupDto : externalLdapGroupInfo) { + String groupName = groupDto.getGroupName(); + if (internalGroupsMap.containsKey(groupName)) { + final Group group = internalGroupsMap.get(groupName); + if (!group.isLdapGroup()) { + batchInfo.getGroupsToBecomeLdap().add(groupName); + } + internalGroupsMap.remove(groupName); + } else { + batchInfo.getGroupsToBeCreated().add(groupName); + } + refreshGroupMembers(batchInfo, groupDto, internalUsersMap); } - for (String user : users) { - if (!externalUsersMap.containsKey(user)) { - throw new AmbariException("Couldn't sync LDAP user " + user - + ", it doesn't exist"); + for (Entry<String, Group> internalGroup : internalGroupsMap.entrySet()) { + if (internalGroup.getValue().isLdapGroup()) { + batchInfo.getGroupsToBeRemoved().add(internalGroup.getValue().getGroupName()); } } - final Set<LdapGroupDto> externalGroups = getExternalLdapGroupInfo(); - final Map<String, LdapGroupDto> externalGroupsMap = new HashMap<String, LdapGroupDto>(); - for (LdapGroupDto group: externalGroups) { - externalGroupsMap.put(group.getGroupName(), group); + + return batchInfo; + } + + /** + * Performs synchronization of given sets of all users. + * + * @throws AmbariException if synchronization failed for any reason + */ + public LdapBatchDto synchronizeAllLdapUsers(LdapBatchDto batchInfo) throws AmbariException { + + Set<LdapUserDto> externalLdapUserInfo = getExternalLdapUserInfo(); + Map<String, User> internalUsersMap = getInternalUsers(); + + for (LdapUserDto userDto : externalLdapUserInfo) { + String userName = userDto.getUserName(); + if (internalUsersMap.containsKey(userName)) { + final User user = internalUsersMap.get(userName); + if (user != null && !user.isLdapUser()) { + batchInfo.getUsersToBecomeLdap().add(userName); + } + internalUsersMap.remove(userName); + } else { + batchInfo.getUsersToBeCreated().add(userName); + } + } + for (Entry<String, User> internalUser : internalUsersMap.entrySet()) { + if (internalUser.getValue().isLdapUser()) { + batchInfo.getUsersToBeRemoved().add(internalUser.getValue().getUserName()); + } } + + return batchInfo; + } + + /** + * Performs synchronization of given set of groupnames. + * + * @param groups set of groups to synchronize + * @throws AmbariException if synchronization failed for any reason + */ + public LdapBatchDto synchronizeLdapGroups(Set<String> groups, LdapBatchDto batchInfo) throws AmbariException { + + final Set<LdapGroupDto> specifiedGroups = new HashSet<LdapGroupDto>(); for (String group : groups) { - if (!externalGroupsMap.containsKey(group)) { + Set<LdapGroupDto> groupDtos = getLdapGroups(group); + if (groupDtos.isEmpty()) { throw new AmbariException("Couldn't sync LDAP group " + group + ", it doesn't exist"); } + specifiedGroups.addAll(groupDtos); } final Map<String, Group> internalGroupsMap = getInternalGroups(); final Map<String, User> internalUsersMap = getInternalUsers(); - // processing groups - for (String groupName : groups) { + for (LdapGroupDto groupDto : specifiedGroups) { + String groupName = groupDto.getGroupName(); if (internalGroupsMap.containsKey(groupName)) { final Group group = internalGroupsMap.get(groupName); if (!group.isLdapGroup()) { batchInfo.getGroupsToBecomeLdap().add(groupName); } + internalGroupsMap.remove(groupName); } else { batchInfo.getGroupsToBeCreated().add(groupName); } - refreshGroupMembers(batchInfo, externalGroupsMap.get(groupName), internalUsersMap, externalUsers); - internalGroupsMap.remove(groupName); + refreshGroupMembers(batchInfo, groupDto, internalUsersMap); } - for (Entry<String, Group> internalGroup : internalGroupsMap.entrySet()) { - if (internalGroup.getValue().isLdapGroup()) { - batchInfo.getGroupsToBeRemoved().add(internalGroup.getValue().getGroupName()); + + return batchInfo; + } + + /** + * Performs synchronization of given set of usernames. + * + * @param users set of users to synchronize + * @throws AmbariException if synchronization failed for any reason + */ + public LdapBatchDto synchronizeLdapUsers(Set<String> users, LdapBatchDto batchInfo) throws AmbariException { + + final Set<LdapUserDto> specifiedUsers = new HashSet<LdapUserDto>(); + + for (String user : users) { + Set<LdapUserDto> userDtos = getLdapUsers(user); + if (userDtos.isEmpty()) { + throw new AmbariException("Couldn't sync LDAP user " + user + + ", it doesn't exist"); } + specifiedUsers.addAll(userDtos); } - // processing users - for (String userName : users) { + final Map<String, User> internalUsersMap = getInternalUsers(); + for (LdapUserDto userDto : specifiedUsers) { + String userName = userDto.getUserName(); if (internalUsersMap.containsKey(userName)) { final User user = internalUsersMap.get(userName); if (user != null && !user.isLdapUser()) { batchInfo.getUsersToBecomeLdap().add(userName); } + internalUsersMap.remove(userName); } else { batchInfo.getUsersToBeCreated().add(userName); } @@ -215,32 +287,74 @@ public class AmbariLdapDataPopulator { } /** + * Performs synchronization of existent users and groups. + * + * @throws AmbariException if synchronization failed for any reason + */ + public LdapBatchDto synchronizeExistingLdapGroups(LdapBatchDto batchInfo) throws AmbariException { + final Map<String, Group> internalGroupsMap = getInternalGroups(); + final Map<String, User> internalUsersMap = getInternalUsers(); + + for (Group group : internalGroupsMap.values()) { + if (group.isLdapGroup()) { + Set<LdapGroupDto> groupDtos = getLdapGroups(group.getGroupName()); + if (groupDtos.isEmpty()) { + batchInfo.getGroupsToBeRemoved().add(group.getGroupName()); + } else { + LdapGroupDto groupDto = groupDtos.iterator().next(); + refreshGroupMembers(batchInfo, groupDto, internalUsersMap); + } + } + } + + return batchInfo; + } + + /** + * Performs synchronization of existent users and groups. + * + * @throws AmbariException if synchronization failed for any reason + */ + public LdapBatchDto synchronizeExistingLdapUsers(LdapBatchDto batchInfo) throws AmbariException { + final Map<String, User> internalUsersMap = getInternalUsers(); + + for (User user : internalUsersMap.values()) { + if (user.isLdapUser()) { + Set<LdapUserDto> userDtos = getLdapUsers(user.getUserName()); + if (userDtos.isEmpty()) { + batchInfo.getUsersToBeRemoved().add(user.getUserName()); + } + } + } + + return batchInfo; + } + + /** * Check group members of the synced group: add missing ones and remove the ones absent in external LDAP. * - * @param groupName group name + * @param batchInfo batch update object + * @param group ldap group * @param internalUsers map of internal users - * @param externalUsers set of external users * @throws AmbariException if group refresh failed */ - protected void refreshGroupMembers(LdapBatchDto batchInfo, LdapGroupDto group, Map<String, User> internalUsers, Set<LdapUserDto> externalUsers) throws AmbariException { - final Set<String> externalMembers = new HashSet<String>(); + protected void refreshGroupMembers(LdapBatchDto batchInfo, LdapGroupDto group, Map<String, User> internalUsers) throws AmbariException { + Set<String> externalMembers = new HashSet<String>(); for (String memberAttribute: group.getMemberAttributes()) { - for (LdapUserDto externalUser: externalUsers) { - // memberAttribute may be either DN or UID, check both - if (externalUser.getDn().equals(memberAttribute) || externalUser.getUid().equals(memberAttribute)) { - externalMembers.add(externalUser.getUserName()); - break; - } + LdapUserDto groupMember = getLdapUserByMemberAttr(memberAttribute); + if (groupMember != null) { + externalMembers.add(groupMember.getUserName()); } } - final Map<String, User> internalMembers = getInternalMembers(group.getGroupName()); + String groupName = group.getGroupName(); + final Map<String, User> internalMembers = getInternalMembers(groupName); for (String externalMember: externalMembers) { if (internalUsers.containsKey(externalMember)) { final User user = internalUsers.get(externalMember); if (user == null) { // user is fresh and is already added to batch info if (!internalMembers.containsKey(externalMember)) { - batchInfo.getMembershipToAdd().add(new LdapUserGroupMemberDto(group.getGroupName(), externalMember)); + batchInfo.getMembershipToAdd().add(new LdapUserGroupMemberDto(groupName, externalMember)); } continue; } @@ -248,21 +362,47 @@ public class AmbariLdapDataPopulator { batchInfo.getUsersToBecomeLdap().add(externalMember); } if (!internalMembers.containsKey(externalMember)) { - batchInfo.getMembershipToAdd().add(new LdapUserGroupMemberDto(group.getGroupName(), externalMember)); + batchInfo.getMembershipToAdd().add(new LdapUserGroupMemberDto(groupName, externalMember)); } internalMembers.remove(externalMember); } else { batchInfo.getUsersToBeCreated().add(externalMember); - batchInfo.getMembershipToAdd().add(new LdapUserGroupMemberDto(group.getGroupName(), externalMember)); - internalUsers.put(externalMember, null); + batchInfo.getMembershipToAdd().add(new LdapUserGroupMemberDto(groupName, externalMember)); } } for (Entry<String, User> userToBeUnsynced: internalMembers.entrySet()) { final User user = userToBeUnsynced.getValue(); - batchInfo.getMembershipToRemove().add(new LdapUserGroupMemberDto(group.getGroupName(), user.getUserName())); + batchInfo.getMembershipToRemove().add(new LdapUserGroupMemberDto(groupName, user.getUserName())); } } + protected Set<LdapGroupDto> getLdapGroups(String groupName) { + Filter groupObjectFilter = new EqualsFilter("objectClass", + ldapServerProperties.getGroupObjectClass()); + Filter groupNameFilter = new LikeFilter(ldapServerProperties.getGroupNamingAttr(), groupName); + Set<LdapGroupDto> filteredLdapGroups = getFilteredLdapGroups(groupObjectFilter, groupNameFilter); + return filteredLdapGroups; + } + + protected Set<LdapUserDto> getLdapUsers(String username) { + Filter userObjectFilter = new EqualsFilter("objectClass", ldapServerProperties.getUserObjectClass()); + Filter userNameFilter = new LikeFilter(ldapServerProperties.getUsernameAttribute(), username); + Set<LdapUserDto> filteredLdapUsers = getFilteredLdapUsers(userObjectFilter, userNameFilter); + return filteredLdapUsers; + } + + protected LdapUserDto getLdapUserByMemberAttr(String memberAttribute) { + // memberAttribute may be either DN or UID, check both + Filter userObjectFilter = new EqualsFilter("objectClass", ldapServerProperties.getUserObjectClass()); + Filter dnFilter = new EqualsFilter("dn", memberAttribute); + Filter uidFilter = new EqualsFilter("uid", memberAttribute); + OrFilter orFilter = new OrFilter(); + orFilter.or(dnFilter); + orFilter.or(uidFilter); + Set<LdapUserDto> filteredLdapUsers = getFilteredLdapUsers(userObjectFilter, orFilter); + return (filteredLdapUsers.isEmpty()) ? null : filteredLdapUsers.iterator().next(); + } + /** * Removes synced users which are not present in any of group. * @@ -285,12 +425,24 @@ public class AmbariLdapDataPopulator { * @return set of info about LDAP groups */ protected Set<LdapGroupDto> getExternalLdapGroupInfo() { + EqualsFilter groupObjectFilter = new EqualsFilter("objectClass", + ldapServerProperties.getGroupObjectClass()); + return getFilteredLdapGroups(groupObjectFilter); + } + + private Set<LdapGroupDto> getFilteredLdapGroups(Filter...filters) { + AndFilter andFilter = new AndFilter(); + for (Filter filter : filters) { + andFilter.and(filter); + } + return getFilteredLdapGroups(andFilter); + } + + private Set<LdapGroupDto> getFilteredLdapGroups(Filter filter) { final Set<LdapGroupDto> groups = new HashSet<LdapGroupDto>(); final LdapTemplate ldapTemplate = loadLdapTemplate(); - final EqualsFilter equalsFilter = new EqualsFilter("objectClass", - ldapServerProperties.getGroupObjectClass()); String baseDn = ldapServerProperties.getBaseDN(); - ldapTemplate.search(baseDn, equalsFilter.encode(), new ContextMapper() { + ldapTemplate.search(baseDn, filter.encode(), new ContextMapper() { @Override public Object mapFromContext(Object ctx) { @@ -320,12 +472,24 @@ public class AmbariLdapDataPopulator { * @return set of info about LDAP users */ protected Set<LdapUserDto> getExternalLdapUserInfo() { + EqualsFilter userObjectFilter = new EqualsFilter("objectClass", + ldapServerProperties.getUserObjectClass()); + return getFilteredLdapUsers(userObjectFilter); + } + + private Set<LdapUserDto> getFilteredLdapUsers(Filter...filters) { + AndFilter andFilter = new AndFilter(); + for (Filter filter : filters) { + andFilter.and(filter); + } + return getFilteredLdapUsers(andFilter); + } + + private Set<LdapUserDto> getFilteredLdapUsers(Filter filter) { final Set<LdapUserDto> users = new HashSet<LdapUserDto>(); final LdapTemplate ldapTemplate = loadLdapTemplate(); - final EqualsFilter equalsFilter = new EqualsFilter("objectClass", - ldapServerProperties.getUserObjectClass()); String baseDn = ldapServerProperties.getBaseDN(); - ldapTemplate.search(baseDn, equalsFilter.encode(), new ContextMapper() { + ldapTemplate.search(baseDn, filter.encode(), new ContextMapper() { @Override public Object mapFromContext(Object ctx) { @@ -337,11 +501,11 @@ public class AmbariLdapDataPopulator { user.setUserName(usernameAttribute.toLowerCase()); user.setUid(uidAttribute.toLowerCase()); user.setDn(adapter.getNameInNamespace().toLowerCase()); + users.add(user); } else { LOG.warn("Ignoring LDAP user " + adapter.getNameInNamespace() + " as it doesn't have required" + " attributes uid and " + ldapServerProperties.getUsernameAttribute()); } - users.add(user); return null; } }); http://git-wip-us.apache.org/repos/asf/ambari/blob/21d784b6/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index 1ad906f..5843279 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -41,6 +41,8 @@ import random import pwd from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException import json +import base64 +from threading import Thread from ambari_commons import OSCheck, OSConst, Firewall from ambari_server import utils @@ -64,6 +66,7 @@ UPGRADE_STACK_ACTION = "upgradestack" STATUS_ACTION = "status" SETUP_HTTPS_ACTION = "setup-https" LDAP_SETUP_ACTION = "setup-ldap" +LDAP_SYNC_ACTION = "sync-ldap" SETUP_GANGLIA_HTTPS_ACTION = "setup-ganglia-https" SETUP_NAGIOS_HTTPS_ACTION = "setup-nagios-https" ENCRYPT_PASSWORDS_ACTION = "encrypt-passwords" @@ -119,6 +122,14 @@ SERVER_LOG_FILE = "/var/log/ambari-server/ambari-server.log" BLIND_PASSWORD = "*****" ROOT_FS_PATH = "/" +# api properties +SERVER_API_HOST = '127.0.0.1' +SERVER_API_PROTOCOL = 'http' +SERVER_API_PORT = '8080' +SERVER_API_LDAP_URL = '/api/v1/controllers/ldap' +SERVER_API_LOGIN = 'admin' +SERVER_API_PASS = 'admin' + # terminal styles BOLD_ON = '\033[1m' BOLD_OFF = '\033[0m' @@ -2994,6 +3005,67 @@ def get_prompt_default(defaultStr=None): else: return '(' + defaultStr + ')' +def sync_ldap(): + if not is_root(): + err = 'Ambari-server sync-ldap should be run with ' \ + 'root-level privileges' + raise FatalException(4, err) + + server_status, pid = is_server_runing() + if not server_status: + err = 'Ambari Server is not running.' + raise FatalException(1, err) + + ldap_configured = get_ambari_properties().get_property(IS_LDAP_CONFIGURED) + if ldap_configured != 'true': + err = "LDAP is not configured. Run 'ambari-server setup-ldap' first." + raise FatalException(1, err) + + url = '{0}://{1}:{2!s}{3}'.format(SERVER_API_PROTOCOL, SERVER_API_HOST, SERVER_API_PORT, SERVER_API_LDAP_URL) + admin_auth = base64.encodestring('%s:%s' % (SERVER_API_LOGIN, SERVER_API_PASS)).replace('\n', '') + request = urllib2.Request(url) + request.add_header('Authorization', 'Basic %s' % admin_auth) + request.add_header('X-Requested-By', 'ambari') + body = [{"LDAP":{"synced_groups":"*","synced_users":"*"}}] + request.add_data(json.dumps(body)) + request.get_method = lambda: 'PUT' + progress_message_thread = None + request_in_progress = True + def print_progress(message): + sys.stdout.write(message) + sys.stdout.flush() + while request_in_progress: + sys.stdout.write('.') + sys.stdout.flush() + time.sleep(1) + sys.stdout.write('\n') + sys.stdout.flush() + try: + progress_message_thread = Thread(target=print_progress, args=('Syncing Ambari Database for permissions for the LDAP Users and Groups..',)) + progress_message_thread.start() + response = urllib2.urlopen(request) + except Exception as e: + err = 'Sync failed. Error details: %s' % e + raise FatalException(1, err) + finally: + request_in_progress = False + if progress_message_thread is not None: + progress_message_thread.join() + response_status_code = response.getcode() + if response_status_code != 200: + err = 'Error during syncing. Http status code - ' + response_status_code + raise FatalException(1, err) + response_body = json.loads(response.read()) + sync_info = response_body['resources'][0]['Sync'] + if sync_info['status'] != 'successful': + raise FatalException(1, sync_info['summary']) + else: + print 'Synced:' + for principal_type, summary in sync_info['summary'].iteritems(): + print ' {0}:'.format(principal_type) + for action, amount in summary.iteritems(): + print ' - {0} = {1!s}'.format(action, amount) + print 'Finished LDAP Sync.' def setup_ldap(): if not is_root(): @@ -4343,6 +4415,8 @@ def main(): upgrade_stack(options, stack_id, repo_url, repo_url_os) elif action == LDAP_SETUP_ACTION: setup_ldap() + elif action == LDAP_SYNC_ACTION: + sync_ldap() elif action == SETUP_SECURITY_ACTION: need_restart = setup_security(options) elif action == REFRESH_STACK_HASH_ACTION: