http://git-wip-us.apache.org/repos/asf/ranger/blob/d424b1a8/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java new file mode 100644 index 0000000..6a21eff --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java @@ -0,0 +1,1188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.patch; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.log4j.Logger; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.biz.PolicyRefUpdater; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.db.XXGroupDao; +import org.apache.ranger.db.XXPolicyDao; +import org.apache.ranger.db.XXPolicyRefAccessTypeDao; +import org.apache.ranger.db.XXPolicyRefConditionDao; +import org.apache.ranger.db.XXPolicyRefDataMaskTypeDao; +import org.apache.ranger.db.XXPolicyRefGroupDao; +import org.apache.ranger.db.XXPolicyRefResourceDao; +import org.apache.ranger.db.XXPolicyRefUserDao; +import org.apache.ranger.db.XXUserDao; +import org.apache.ranger.entity.XXAccessTypeDef; +import org.apache.ranger.entity.XXDataMaskTypeDef; +import org.apache.ranger.entity.XXGroup; +import org.apache.ranger.entity.XXPolicy; +import org.apache.ranger.entity.XXPolicyConditionDef; +import org.apache.ranger.entity.XXPolicyItem; +import org.apache.ranger.entity.XXPolicyItemAccess; +import org.apache.ranger.entity.XXPolicyItemCondition; +import org.apache.ranger.entity.XXPolicyItemDataMaskInfo; +import org.apache.ranger.entity.XXPolicyItemGroupPerm; +import org.apache.ranger.entity.XXPolicyItemRowFilterInfo; +import org.apache.ranger.entity.XXPolicyItemUserPerm; +import org.apache.ranger.entity.XXPolicyLabel; +import org.apache.ranger.entity.XXPolicyLabelMap; +import org.apache.ranger.entity.XXPolicyRefAccessType; +import org.apache.ranger.entity.XXPolicyRefCondition; +import org.apache.ranger.entity.XXPolicyRefDataMaskType; +import org.apache.ranger.entity.XXPolicyRefGroup; +import org.apache.ranger.entity.XXPolicyRefResource; +import org.apache.ranger.entity.XXPolicyRefUser; +import org.apache.ranger.entity.XXPolicyResource; +import org.apache.ranger.entity.XXPolicyResourceMap; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.entity.XXUser; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerValiditySchedule; +import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator; +import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.service.RangerPolicyService; +import org.apache.ranger.util.CLIUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; + +/** + * Consolidates Ranger policy details into a JSON string and stores it into a + * column in x_policy table After running this patch Ranger policy can be + * completely read/saved into x_policy table and some related Ref tables (which + * maintain ID->String mapping for each policy). + * + */ +@Component +public class PatchForUpdatingPolicyJson_J10019 extends BaseLoader { + private static final Logger logger = Logger.getLogger(PatchForUpdatingPolicyJson_J10019.class); + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + ServiceDBStore svcStore; + + @Autowired + @Qualifier(value = "transactionManager") + PlatformTransactionManager txManager; + + @Autowired + PolicyRefUpdater policyRefUpdater; + + private final Map<String, Long> groupIdMap = new HashMap<>(); + private final Map<String, Long> userIdMap = new HashMap<>(); + private final Map<String, Map<String, Long>> resourceNameIdMap = new HashMap<>(); + private final Map<String, Map<String, Long>> accessTypeIdMap = new HashMap<>(); + private final Map<String, Map<String, Long>> conditionNameIdMap = new HashMap<>(); + private final Map<String, Map<String, Long>> dataMaskTypeIdMap = new HashMap<>(); + + public static void main(String[] args) { + logger.info("main()"); + try { + PatchForUpdatingPolicyJson_J10019 loader = (PatchForUpdatingPolicyJson_J10019) CLIUtil.getBean(PatchForUpdatingPolicyJson_J10019.class); + + loader.init(); + + while (loader.isMoreToProcess()) { + loader.load(); + } + + logger.info("Load complete. Exiting!!!"); + + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForUpdatingPolicyJson.execLoad()"); + + try { + updateRangerPolicyTableWithPolicyJson(); + } catch (Exception e) { + logger.error("Error while updateRangerPolicyTableWithPolicyJson()", e); + System.exit(1); + } + + logger.info("<== PatchForUpdatingPolicyJson.execLoad()"); + } + + @Override + public void printStats() { + logger.info("updateRangerPolicyTableWithPolicyJson data "); + } + + private void updateRangerPolicyTableWithPolicyJson() throws Exception { + logger.info("==> updateRangerPolicyTableWithPolicyJson() "); + + List<RangerService> allServices = svcStore.getServices(new SearchFilter()); + + if (CollectionUtils.isNotEmpty(allServices)) { + for (RangerService service : allServices) { + XXService dbService = daoMgr.getXXService().getById(service.getId()); + + logger.info("==> Port Policies of service(name=" + dbService.getName() + ")"); + + RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager); + + List<RangerPolicy> policies = policyRetriever.getServicePolicies(dbService); + + if (CollectionUtils.isNotEmpty(policies)) { + for (RangerPolicy policy : policies) { + policyRefUpdater.cleanupRefTables(policy); + portPolicy(service.getType(), policy); + } + } + } + } + + logger.info("<== updateRangerPolicyTableWithPolicyJson() "); + } + + private void portPolicy(String serviceType, RangerPolicy policy) throws Exception { + logger.info("==> portPolicy(id=" + policy.getId() + ")"); + + String policyText = JsonUtils.objectToJson(policy); + + if (StringUtils.isEmpty(policyText)) { + throw new Exception("Failed to convert policy to json string. Policy: [id=" + policy.getId() + "; name=" + policy.getName() + "; serviceType=" + serviceType + "]"); + } + + XXPolicyDao policyDao = daoMgr.getXXPolicy(); + XXPolicy dbBean = policyDao.getById(policy.getId()); + + dbBean.setPolicyText(policyText); + + policyDao.update(dbBean); + + try { + Set<String> accesses = new HashSet<>(); + Set<String> users = new HashSet<>(); + Set<String> groups = new HashSet<>(); + Set<String> conditions = new HashSet<>(); + Set<String> dataMasks = new HashSet<>(); + + buildLists(policy.getPolicyItems(), accesses, conditions, users, groups); + buildLists(policy.getDenyPolicyItems(), accesses, conditions, users, groups); + buildLists(policy.getAllowExceptions(), accesses, conditions, users, groups); + buildLists(policy.getDenyExceptions(), accesses, conditions, users, groups); + buildLists(policy.getDataMaskPolicyItems(), accesses, conditions, users, groups); + buildLists(policy.getRowFilterPolicyItems(), accesses, conditions, users, groups); + + buildList(policy.getDataMaskPolicyItems(), dataMasks); + + addResourceDefRef(serviceType, policy); + addUserNameRef(policy.getId(), users); + addGroupNameRef(policy.getId(), groups); + addAccessDefRef(serviceType, policy.getId(), accesses); + addPolicyConditionDefRef(serviceType, policy.getId(), conditions); + addDataMaskDefRef(serviceType, policy.getId(), dataMasks); + } catch (Exception e) { + logger.error("portPoliry(id=" + policy.getId() +") failed!!"); + logger.error("Offending policy:" + policyText); + throw e; + } + + logger.info("<== portPolicy(id=" + policy.getId() + ")"); + } + + private void addResourceDefRef(String serviceType, RangerPolicy policy) throws Exception { + logger.info("==> addResourceDefRef(id=" + policy.getId() + ")"); + + Map<String, Long> serviceDefResourceNameIDMap = resourceNameIdMap.get(serviceType); + + if (serviceDefResourceNameIDMap == null) { + serviceDefResourceNameIDMap = new HashMap<>(); + + resourceNameIdMap.put(serviceType, serviceDefResourceNameIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXResourceDef resourceDef : daoMgr.getXXResourceDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefResourceNameIDMap.put(resourceDef.getName(), resourceDef.getId()); + } + } + + Map<String, RangerPolicyResource> policyResources = policy.getResources(); + + if (MapUtils.isNotEmpty(policyResources)) { + XXPolicyRefResourceDao policyRefResourceDao = daoMgr.getXXPolicyRefResource(); + Set<String> resourceNames = policyResources.keySet(); + + for (String resourceName : resourceNames) { + Long resourceDefId = serviceDefResourceNameIDMap.get(resourceName); + + if (resourceDefId == null) { + throw new Exception(resourceName + ": unknown resource in policy [id=" + policy.getId() + "; name=" + policy.getName() + "; serviceType=" + serviceType + "]. Known resources: " + serviceDefResourceNameIDMap.keySet()); + } + + // insert policy-id, resourceDefId, resourceName into Ref table + XXPolicyRefResource policyRefResource = new XXPolicyRefResource(); + + policyRefResource.setPolicyId(policy.getId()); + policyRefResource.setResourceDefId(resourceDefId); + policyRefResource.setResourceName(resourceName); + + policyRefResourceDao.create(policyRefResource); + } + } + + logger.info("<== addResourceDefRef(id=" + policy.getId() + ")"); + } + + private void addUserNameRef(Long policyId, Set<String> users) throws Exception { + logger.info("==> addUserNameRef(id=" + policyId + ")"); + + XXPolicyRefUserDao policyRefUserDao = daoMgr.getXXPolicyRefUser(); + XXUserDao userDao = daoMgr.getXXUser(); + + // insert policy-id, userName into Ref table + for (String user : users) { + Long userId = userIdMap.get(user); + + if (userId == null) { + XXUser userObject = userDao.findByUserName(user); + + if (userObject == null) { + throw new Exception(user + ": unknown user in policy [id=" + policyId + "]"); + } + + userId = userObject.getId(); + + userIdMap.put(user, userId); + } + + XXPolicyRefUser policyRefUser = new XXPolicyRefUser(); + + policyRefUser.setPolicyId(policyId); + policyRefUser.setUserName(user); + policyRefUser.setUserId(userId); + + policyRefUserDao.create(policyRefUser); + } + + logger.info("<== addUserNameRef(id=" + policyId + ")"); + } + + private void addGroupNameRef(Long policyId, Set<String> groups) throws Exception { + logger.info("==> addGroupNameRef(id=" + policyId + ")"); + + // insert policy-id, groupName into Ref table + XXPolicyRefGroupDao policyRefGroupDao = daoMgr.getXXPolicyRefGroup(); + XXGroupDao groupDao = daoMgr.getXXGroup(); + + for (String group : groups) { + Long groupId = groupIdMap.get(group); + + if (groupId == null) { + XXGroup groupObject = groupDao.findByGroupName(group); + + if (groupObject == null) { + throw new Exception(group + ": unknown group in policy [id=" + policyId + "]"); + } + + groupId = groupObject.getId(); + + groupIdMap.put(group, groupId); + } + + XXPolicyRefGroup policyRefGroup = new XXPolicyRefGroup(); + + policyRefGroup.setPolicyId(policyId); + policyRefGroup.setGroupName(group); + policyRefGroup.setGroupId(groupId); + + policyRefGroupDao.create(policyRefGroup); + } + + logger.info("<== addGroupNameRef(id=" + policyId + ")"); + + } + + private void addAccessDefRef(String serviceType, Long policyId, Set<String> accesses) throws Exception { + logger.info("==> addAccessDefRef(id=" + policyId + ")"); + // insert policy-id, accessName into Ref table + + Map<String, Long> serviceDefAccessTypeIDMap = accessTypeIdMap.get(serviceType); + + if (serviceDefAccessTypeIDMap == null) { + serviceDefAccessTypeIDMap = new HashMap<>(); + + accessTypeIdMap.put(serviceType, serviceDefAccessTypeIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXAccessTypeDef accessTypeDef : daoMgr.getXXAccessTypeDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefAccessTypeIDMap.put(accessTypeDef.getName(), accessTypeDef.getId()); + } + } + + XXPolicyRefAccessTypeDao policyRefAccessTypeDao = daoMgr.getXXPolicyRefAccessType(); + + for (String access : accesses) { + Long accessTypeDefId = serviceDefAccessTypeIDMap.get(access); + + if (accessTypeDefId == null) { + throw new Exception(access + ": unknown accessType in policy [id=" + policyId + "; serviceType=" + serviceType + "]. Known accessTypes: " + serviceDefAccessTypeIDMap.keySet()); + } + + XXPolicyRefAccessType policyRefAccessType = new XXPolicyRefAccessType(); + + policyRefAccessType.setPolicyId(policyId); + policyRefAccessType.setAccessTypeName(access); + policyRefAccessType.setAccessDefId(accessTypeDefId); + + policyRefAccessTypeDao.create(policyRefAccessType); + } + + logger.info("<== addAccessDefRef(id=" + policyId + ")"); + } + + private void addPolicyConditionDefRef(String serviceType, Long policyId, Set<String> conditions) throws Exception { + logger.info("==> addPolicyConditionDefRef(id=" + policyId + ")"); + // insert policy-id, conditionName into Ref table + + Map<String, Long> serviceDefConditionNameIDMap = conditionNameIdMap.get(serviceType); + + if (serviceDefConditionNameIDMap == null) { + serviceDefConditionNameIDMap = new HashMap<>(); + + conditionNameIdMap.put(serviceType, serviceDefConditionNameIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXPolicyConditionDef conditionDef : daoMgr.getXXPolicyConditionDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefConditionNameIDMap.put(conditionDef.getName(), conditionDef.getId()); + } + } + + XXPolicyRefConditionDao policyRefConditionDao = daoMgr.getXXPolicyRefCondition(); + + for (String condition : conditions) { + Long conditionDefId = serviceDefConditionNameIDMap.get(condition); + + if (conditionDefId == null) { + throw new Exception(condition + ": unknown condition in policy [id=" + policyId + "; serviceType=" + serviceType + "]. Known conditions are: " + serviceDefConditionNameIDMap.keySet()); + } + + XXPolicyRefCondition policyRefCondition = new XXPolicyRefCondition(); + + policyRefCondition.setPolicyId(policyId); + policyRefCondition.setConditionName(condition); + policyRefCondition.setConditionDefId(conditionDefId); + + policyRefConditionDao.create(policyRefCondition); + } + + logger.info("<== addPolicyConditionDefRef(id=" + policyId + ")"); + } + + private void addDataMaskDefRef(String serviceType, Long policyId, Set<String> datamasks) throws Exception { + logger.info("==> addDataMaskDefRef(id=" + policyId + ")"); + + // insert policy-id, datamaskName into Ref table + + Map<String, Long> serviceDefDataMaskTypeIDMap = dataMaskTypeIdMap.get(serviceType); + + if (serviceDefDataMaskTypeIDMap == null) { + serviceDefDataMaskTypeIDMap = new HashMap<>(); + + dataMaskTypeIdMap.put(serviceType, serviceDefDataMaskTypeIDMap); + + XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType); + + for (XXDataMaskTypeDef dataMaskTypeDef : daoMgr.getXXDataMaskTypeDef().findByServiceDefId(dbServiceDef.getId())) { + serviceDefDataMaskTypeIDMap.put(dataMaskTypeDef.getName(), dataMaskTypeDef.getId()); + } + } + + XXPolicyRefDataMaskTypeDao policyRefDataMaskTypeDao = daoMgr.getXXPolicyRefDataMaskType(); + + for (String datamask : datamasks) { + Long dataMaskTypeId = serviceDefDataMaskTypeIDMap.get(datamask); + + if (dataMaskTypeId == null) { + throw new Exception(datamask + ": unknown dataMaskType in policy [id=" + policyId + "; serviceType=" + serviceType + "]. Known dataMaskTypes " + serviceDefDataMaskTypeIDMap.keySet()); + } + + XXPolicyRefDataMaskType policyRefDataMaskType = new XXPolicyRefDataMaskType(); + + policyRefDataMaskType.setPolicyId(policyId); + policyRefDataMaskType.setDataMaskTypeName(datamask); + policyRefDataMaskType.setDataMaskDefId(dataMaskTypeId); + + policyRefDataMaskTypeDao.create(policyRefDataMaskType); + } + + logger.info("<== addDataMaskDefRef(id=" + policyId + ")"); + + } + + private void buildLists(List<? extends RangerPolicyItem> policyItems, Set<String> accesses, Set<String> conditions, Set<String> users, Set<String> groups) { + for (RangerPolicyItem item : policyItems) { + for (RangerPolicyItemAccess policyAccess : item.getAccesses()) { + accesses.add(policyAccess.getType()); + } + + for (RangerPolicyItemCondition policyCondition : item.getConditions()) { + conditions.add(policyCondition.getType()); + } + + users.addAll(item.getUsers()); + groups.addAll(item.getGroups()); + } + } + + private void buildList(List<RangerDataMaskPolicyItem> dataMaskPolicyItems, Set<String> dataMasks) { + for (RangerDataMaskPolicyItem datMaskPolicyItem : dataMaskPolicyItems) { + dataMasks.add(datMaskPolicyItem.getDataMaskInfo().getDataMaskType()); + } + } + + static private class RangerPolicyRetriever { + static final Log LOG = LogFactory.getLog(RangerPolicyRetriever.class); + static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever"); + + private final RangerDaoManager daoMgr; + private final LookupCache lookupCache = new LookupCache(); + + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + + RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) { + this.daoMgr = daoMgr; + this.txManager = txManager; + + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } + } + + private class PolicyLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + List<RangerPolicy> policies; + + PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + public List<RangerPolicy> getPolicies() { + return policies; + } + + @Override + public void run() { + txTemplate.setReadOnly(true); + policies = txTemplate.execute(new TransactionCallback<List<RangerPolicy>>() { + @Override + public List<RangerPolicy> doInTransaction(TransactionStatus status) { + RetrieverContext ctx = new RetrieverContext(xService); + return ctx.getAllPolicies(); + } + }); + } + } + + public List<RangerPolicy> getServicePolicies(final XXService xService) throws InterruptedException { + String serviceName = xService == null ? null : xService.getName(); + Long serviceId = xService == null ? null : xService.getId(); + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ", serviceId=" + serviceId + ")"); + } + + List<RangerPolicy> ret = null; + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ",serviceId=" + serviceId + ")"); + } + + if (xService != null) { + if (txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction"); + } + + RetrieverContext ctx = new RetrieverContext(xService); + + ret = ctx.getAllPolicies(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Retrieving policies in a new, read-only transaction"); + } + + PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService); + t.start(); + t.join(); + ret = t.getPolicies(); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter"); + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ", serviceId=" + serviceId + "): policyCount=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + class LookupCache { + final Map<Long, String> userNames = new HashMap<Long, String>(); + final Map<Long, String> userScreenNames = new HashMap<Long, String>(); + final Map<Long, String> groupNames = new HashMap<Long, String>(); + final Map<Long, String> accessTypes = new HashMap<Long, String>(); + final Map<Long, String> conditions = new HashMap<Long, String>(); + final Map<Long, String> resourceDefs = new HashMap<Long, String>(); + final Map<Long, String> dataMasks = new HashMap<Long, String>(); + final Map<Long, String> policyLabels = new HashMap<Long, String>(); + + String getUserName(Long userId) { + String ret = null; + + if (userId != null) { + ret = userNames.get(userId); + + if (ret == null) { + XXUser user = daoMgr.getXXUser().getById(userId); + + if (user != null) { + ret = user.getName(); // Name is `loginId` + + userNames.put(userId, ret); + } + } + } + + return ret; + } + + String getPolicyLabelName(Long policyLabelId) { + String ret = null; + + if (policyLabelId != null) { + ret = policyLabels.get(policyLabelId); + + if (ret == null) { + XXPolicyLabel xxPolicyLabel = daoMgr.getXXPolicyLabels().getById(policyLabelId); + + if (xxPolicyLabel != null) { + ret = xxPolicyLabel.getPolicyLabel(); + + policyLabels.put(policyLabelId, ret); + } + } + } + + return ret; + } + + String getUserScreenName(Long userId) { + String ret = null; + + if (userId != null) { + ret = userScreenNames.get(userId); + + if (ret == null) { + XXPortalUser user = daoMgr.getXXPortalUser().getById(userId); + + if (user != null) { + ret = user.getPublicScreenName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getFirstName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getLoginId(); + } else { + if (!StringUtil.isEmpty(user.getLastName())) { + ret += (" " + user.getLastName()); + } + } + } + + if (ret != null) { + userScreenNames.put(userId, ret); + } + } + } + } + + return ret; + } + + String getGroupName(Long groupId) { + String ret = null; + + if (groupId != null) { + ret = groupNames.get(groupId); + + if (ret == null) { + XXGroup group = daoMgr.getXXGroup().getById(groupId); + + if (group != null) { + ret = group.getName(); + + groupNames.put(groupId, ret); + } + } + } + + return ret; + } + + String getAccessType(Long accessTypeId) { + String ret = null; + + if (accessTypeId != null) { + ret = accessTypes.get(accessTypeId); + + if (ret == null) { + XXAccessTypeDef xAccessType = daoMgr.getXXAccessTypeDef().getById(accessTypeId); + + if (xAccessType != null) { + ret = xAccessType.getName(); + + accessTypes.put(accessTypeId, ret); + } else { + LOG.warn("getAccessType(): Canot find name for accessTypeId " + accessTypeId + ". This will cause Ranger policy migration to fail. Please check if all service-defs are migrated correctly!"); + } + } + } + + return ret; + } + + String getConditionType(Long conditionDefId) { + String ret = null; + + if (conditionDefId != null) { + ret = conditions.get(conditionDefId); + + if (ret == null) { + XXPolicyConditionDef xPolicyConditionDef = daoMgr.getXXPolicyConditionDef() + .getById(conditionDefId); + + if (xPolicyConditionDef != null) { + ret = xPolicyConditionDef.getName(); + + conditions.put(conditionDefId, ret); + } + } + } + + return ret; + } + + String getResourceName(Long resourceDefId) { + String ret = null; + + if (resourceDefId != null) { + ret = resourceDefs.get(resourceDefId); + + if (ret == null) { + XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); + + if (xResourceDef != null) { + ret = xResourceDef.getName(); + + resourceDefs.put(resourceDefId, ret); + } + } + } + + return ret; + } + + String getDataMaskName(Long dataMaskDefId) { + String ret = null; + + if (dataMaskDefId != null) { + ret = dataMasks.get(dataMaskDefId); + + if (ret == null) { + XXDataMaskTypeDef xDataMaskDef = daoMgr.getXXDataMaskTypeDef().getById(dataMaskDefId); + + if (xDataMaskDef != null) { + ret = xDataMaskDef.getName(); + + dataMasks.put(dataMaskDefId, ret); + } + } + } + + return ret; + } + } + + static List<XXPolicy> asList(XXPolicy policy) { + List<XXPolicy> ret = new ArrayList<XXPolicy>(); + + if (policy != null) { + ret.add(policy); + } + + return ret; + } + + class RetrieverContext { + final XXService service; + final ListIterator<XXPolicy> iterPolicy; + final ListIterator<XXPolicyResource> iterResources; + final ListIterator<XXPolicyResourceMap> iterResourceMaps; + final ListIterator<XXPolicyItem> iterPolicyItems; + final ListIterator<XXPolicyItemUserPerm> iterUserPerms; + final ListIterator<XXPolicyItemGroupPerm> iterGroupPerms; + final ListIterator<XXPolicyItemAccess> iterAccesses; + final ListIterator<XXPolicyItemCondition> iterConditions; + final ListIterator<XXPolicyItemDataMaskInfo> iterDataMaskInfos; + final ListIterator<XXPolicyItemRowFilterInfo> iterRowFilterInfos; + final ListIterator<XXPolicyLabelMap> iterPolicyLabels; + + RetrieverContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List<XXPolicy> xPolicies = daoMgr.getXXPolicy().findByServiceId(serviceId); + + this.service = xService; + this.iterPolicy = xPolicies.listIterator(); + + List<XXPolicyResource> xResources = daoMgr.getXXPolicyResource().findByServiceId(serviceId); + List<XXPolicyResourceMap> xResourceMaps = daoMgr.getXXPolicyResourceMap().findByServiceId(serviceId); + List<XXPolicyItem> xPolicyItems = daoMgr.getXXPolicyItem().findByServiceId(serviceId); + List<XXPolicyItemUserPerm> xUserPerms = daoMgr.getXXPolicyItemUserPerm().findByServiceId(serviceId); + List<XXPolicyItemGroupPerm> xGroupPerms = daoMgr.getXXPolicyItemGroupPerm().findByServiceId(serviceId); + List<XXPolicyItemAccess> xAccesses = daoMgr.getXXPolicyItemAccess().findByServiceId(serviceId); + List<XXPolicyItemCondition> xConditions = daoMgr.getXXPolicyItemCondition().findByServiceId(serviceId); + List<XXPolicyItemDataMaskInfo> xDataMaskInfos = daoMgr.getXXPolicyItemDataMaskInfo().findByServiceId(serviceId); + List<XXPolicyItemRowFilterInfo> xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByServiceId(serviceId); + List<XXPolicyLabelMap> xPolicyLabelMap = daoMgr.getXXPolicyLabelMap().findByServiceId(serviceId); + + this.iterResources = xResources.listIterator(); + this.iterResourceMaps = xResourceMaps.listIterator(); + this.iterPolicyItems = xPolicyItems.listIterator(); + this.iterUserPerms = xUserPerms.listIterator(); + this.iterGroupPerms = xGroupPerms.listIterator(); + this.iterAccesses = xAccesses.listIterator(); + this.iterConditions = xConditions.listIterator(); + this.iterDataMaskInfos = xDataMaskInfos.listIterator(); + this.iterRowFilterInfos = xRowFilterInfos.listIterator(); + this.iterPolicyLabels = xPolicyLabelMap.listIterator(); + } + + RetrieverContext(XXPolicy xPolicy, XXService xService) { + Long policyId = xPolicy == null ? null : xPolicy.getId(); + List<XXPolicy> xPolicies = asList(xPolicy); + + this.service = xService; + this.iterPolicy = xPolicies.listIterator(); + + List<XXPolicyResource> xResources = daoMgr.getXXPolicyResource().findByPolicyId(policyId); + List<XXPolicyResourceMap> xResourceMaps = daoMgr.getXXPolicyResourceMap().findByPolicyId(policyId); + List<XXPolicyItem> xPolicyItems = daoMgr.getXXPolicyItem().findByPolicyId(policyId); + List<XXPolicyItemUserPerm> xUserPerms = daoMgr.getXXPolicyItemUserPerm().findByPolicyId(policyId); + List<XXPolicyItemGroupPerm> xGroupPerms = daoMgr.getXXPolicyItemGroupPerm().findByPolicyId(policyId); + List<XXPolicyItemAccess> xAccesses = daoMgr.getXXPolicyItemAccess().findByPolicyId(policyId); + List<XXPolicyItemCondition> xConditions = daoMgr.getXXPolicyItemCondition().findByPolicyId(policyId); + List<XXPolicyItemDataMaskInfo> xDataMaskInfos = daoMgr.getXXPolicyItemDataMaskInfo().findByPolicyId(policyId); + List<XXPolicyItemRowFilterInfo> xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByPolicyId(policyId); + List<XXPolicyLabelMap> xPolicyLabelMap = daoMgr.getXXPolicyLabelMap().findByPolicyId(policyId); + + this.iterResources = xResources.listIterator(); + this.iterResourceMaps = xResourceMaps.listIterator(); + this.iterPolicyItems = xPolicyItems.listIterator(); + this.iterUserPerms = xUserPerms.listIterator(); + this.iterGroupPerms = xGroupPerms.listIterator(); + this.iterAccesses = xAccesses.listIterator(); + this.iterConditions = xConditions.listIterator(); + this.iterDataMaskInfos = xDataMaskInfos.listIterator(); + this.iterRowFilterInfos = xRowFilterInfos.listIterator(); + this.iterPolicyLabels = xPolicyLabelMap.listIterator(); + } + + RangerPolicy getNextPolicy() { + RangerPolicy ret = null; + + if (iterPolicy.hasNext()) { + XXPolicy xPolicy = iterPolicy.next(); + + if (xPolicy != null) { + ret = new RangerPolicy(); + + ret.setId(xPolicy.getId()); + ret.setGuid(xPolicy.getGuid()); + ret.setIsEnabled(xPolicy.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xPolicy.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xPolicy.getUpdatedByUserId())); + ret.setCreateTime(xPolicy.getCreateTime()); + ret.setUpdateTime(xPolicy.getUpdateTime()); + ret.setVersion(xPolicy.getVersion()); + ret.setService(service == null ? null : service.getName()); + ret.setName(StringUtils.trim(xPolicy.getName())); + ret.setPolicyType(xPolicy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : xPolicy.getPolicyType()); + ret.setDescription(xPolicy.getDescription()); + ret.setResourceSignature(xPolicy.getResourceSignature()); + ret.setIsAuditEnabled(xPolicy.getIsAuditEnabled()); + ret.setPolicyPriority(xPolicy.getPolicyPriority()); + + Map<String, String> mapOfOptions = JsonUtils.jsonToMapStringString(xPolicy.getOptions()); + + if (MapUtils.isNotEmpty(mapOfOptions)) { + String validitySchedulesStr = mapOfOptions.get(RangerPolicyService.OPTION_POLICY_VALIDITY_SCHEDULES); + + if (StringUtils.isNotEmpty(validitySchedulesStr)) { + List<RangerValiditySchedule> validitySchedules = JsonUtils.jsonToRangerValiditySchedule(validitySchedulesStr); + + ret.setValiditySchedules(validitySchedules); + } + } + + getPolicyLabels(ret); + getResource(ret); + getPolicyItems(ret); + } + } + + return ret; + } + + private void getPolicyLabels(RangerPolicy ret) { + List<String> xPolicyLabels = new ArrayList<String>(); + while (iterPolicyLabels.hasNext()) { + XXPolicyLabelMap xPolicyLabel = iterPolicyLabels.next(); + if (xPolicyLabel.getPolicyId().equals(ret.getId())) { + String policyLabel = lookupCache.getPolicyLabelName(xPolicyLabel.getPolicyLabelId()); + if (policyLabel != null) { + xPolicyLabels.add(policyLabel); + } + ret.setPolicyLabels(xPolicyLabels); + } else { + if (iterPolicyLabels.hasPrevious()) { + iterPolicyLabels.previous(); + } + break; + } + } + } + + List<RangerPolicy> getAllPolicies() { + List<RangerPolicy> ret = new ArrayList<RangerPolicy>(); + + while (iterPolicy.hasNext()) { + RangerPolicy policy = getNextPolicy(); + + if (policy != null) { + ret.add(policy); + } + } + + if (!hasProcessedAll()) { + LOG.warn("getAllPolicies(): perhaps one or more policies got updated during retrieval. Falling back to secondary method"); + + ret = getAllPoliciesBySecondary(); + } + + return ret; + } + + List<RangerPolicy> getAllPoliciesBySecondary() { + List<RangerPolicy> ret = null; + + if (service != null) { + List<XXPolicy> xPolicies = daoMgr.getXXPolicy().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xPolicies)) { + ret = new ArrayList<RangerPolicy>(xPolicies.size()); + + for (XXPolicy xPolicy : xPolicies) { + RetrieverContext ctx = new RetrieverContext(xPolicy, service); + + RangerPolicy policy = ctx.getNextPolicy(); + + if (policy != null) { + ret.add(policy); + } + } + } + } + + return ret; + } + + private boolean hasProcessedAll() { + boolean moreToProcess = iterPolicy.hasNext() || iterResources.hasNext() || iterResourceMaps.hasNext() + || iterPolicyItems.hasNext() || iterUserPerms.hasNext() || iterGroupPerms.hasNext() + || iterAccesses.hasNext() || iterConditions.hasNext() || iterDataMaskInfos.hasNext() + || iterRowFilterInfos.hasNext() || iterPolicyLabels.hasNext(); + + return !moreToProcess; + } + + private void getResource(RangerPolicy policy) { + while (iterResources.hasNext()) { + XXPolicyResource xResource = iterResources.next(); + + if (xResource.getPolicyid().equals(policy.getId())) { + RangerPolicyResource resource = new RangerPolicyResource(); + + resource.setIsExcludes(xResource.getIsexcludes()); + resource.setIsRecursive(xResource.getIsrecursive()); + + while (iterResourceMaps.hasNext()) { + XXPolicyResourceMap xResourceMap = iterResourceMaps.next(); + + if (xResourceMap.getResourceid().equals(xResource.getId())) { + resource.getValues().add(xResourceMap.getValue()); + } else { + if (iterResourceMaps.hasPrevious()) { + iterResourceMaps.previous(); + } + + break; + } + } + + policy.getResources().put(lookupCache.getResourceName(xResource.getResdefid()), resource); + } else if (xResource.getPolicyid().compareTo(policy.getId()) > 0) { + if (iterResources.hasPrevious()) { + iterResources.previous(); + } + + break; + } + } + } + + private void getPolicyItems(RangerPolicy policy) { + while (iterPolicyItems.hasNext()) { + XXPolicyItem xPolicyItem = iterPolicyItems.next(); + + if (xPolicyItem.getPolicyid().equals(policy.getId())) { + final RangerPolicyItem policyItem; + final RangerDataMaskPolicyItem dataMaskPolicyItem; + final RangerRowFilterPolicyItem rowFilterPolicyItem; + + if (xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) { + dataMaskPolicyItem = new RangerDataMaskPolicyItem(); + rowFilterPolicyItem = null; + policyItem = dataMaskPolicyItem; + } else if (xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) { + dataMaskPolicyItem = null; + rowFilterPolicyItem = new RangerRowFilterPolicyItem(); + policyItem = rowFilterPolicyItem; + } else { + dataMaskPolicyItem = null; + rowFilterPolicyItem = null; + policyItem = new RangerPolicyItem(); + } + + while (iterAccesses.hasNext()) { + XXPolicyItemAccess xAccess = iterAccesses.next(); + + if (xAccess.getPolicyitemid().equals(xPolicyItem.getId())) { + policyItem.getAccesses().add(new RangerPolicyItemAccess(lookupCache.getAccessType(xAccess.getType()), xAccess.getIsallowed())); + } else { + if (iterAccesses.hasPrevious()) { + iterAccesses.previous(); + } + + break; + } + } + + while (iterUserPerms.hasNext()) { + XXPolicyItemUserPerm xUserPerm = iterUserPerms.next(); + + if (xUserPerm.getPolicyitemid().equals(xPolicyItem.getId())) { + String userName = lookupCache.getUserName(xUserPerm.getUserid()); + + if (userName != null) { + policyItem.getUsers().add(userName); + } + } else { + if (iterUserPerms.hasPrevious()) { + iterUserPerms.previous(); + } + + break; + } + } + + while (iterGroupPerms.hasNext()) { + XXPolicyItemGroupPerm xGroupPerm = iterGroupPerms.next(); + + if (xGroupPerm.getPolicyitemid().equals(xPolicyItem.getId())) { + String groupName = lookupCache.getGroupName(xGroupPerm.getGroupid()); + + if (groupName != null) { + policyItem.getGroups().add(groupName); + } + } else { + if (iterGroupPerms.hasPrevious()) { + iterGroupPerms.previous(); + } + + break; + } + } + + RangerPolicyItemCondition condition = null; + Long prevConditionType = null; + + while (iterConditions.hasNext()) { + XXPolicyItemCondition xCondition = iterConditions.next(); + + if (xCondition.getPolicyitemid().equals(xPolicyItem.getId())) { + if (!xCondition.getType().equals(prevConditionType)) { + condition = new RangerPolicyItemCondition(); + + condition.setType(lookupCache.getConditionType(xCondition.getType())); + condition.getValues().add(xCondition.getValue()); + + policyItem.getConditions().add(condition); + + prevConditionType = xCondition.getType(); + } else { + condition.getValues().add(xCondition.getValue()); + } + } else { + if (iterConditions.hasPrevious()) { + iterConditions.previous(); + } + + break; + } + } + + policyItem.setDelegateAdmin(xPolicyItem.getDelegateAdmin()); + + if (dataMaskPolicyItem != null) { + while (iterDataMaskInfos.hasNext()) { + XXPolicyItemDataMaskInfo xDataMaskInfo = iterDataMaskInfos.next(); + + if (xDataMaskInfo.getPolicyItemId().equals(xPolicyItem.getId())) { + dataMaskPolicyItem.setDataMaskInfo(new RangerPolicyItemDataMaskInfo(lookupCache.getDataMaskName(xDataMaskInfo.getType()), xDataMaskInfo.getConditionExpr(), xDataMaskInfo.getValueExpr())); + } else { + if (iterDataMaskInfos.hasPrevious()) { + iterDataMaskInfos.previous(); + } + + break; + } + } + } + + if (rowFilterPolicyItem != null) { + while (iterRowFilterInfos.hasNext()) { + XXPolicyItemRowFilterInfo xRowFilterInfo = iterRowFilterInfos.next(); + + if (xRowFilterInfo.getPolicyItemId().equals(xPolicyItem.getId())) { + rowFilterPolicyItem.setRowFilterInfo(new RangerPolicyItemRowFilterInfo(xRowFilterInfo.getFilterExpr())); + } else { + if (iterRowFilterInfos.hasPrevious()) { + iterRowFilterInfos.previous(); + } + + break; + } + } + } + + int itemType = xPolicyItem.getItemType() == null ? RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW : xPolicyItem.getItemType(); + + if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) { + policy.getPolicyItems().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) { + policy.getDenyPolicyItems().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS) { + policy.getAllowExceptions().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS) { + policy.getDenyExceptions().add(policyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) { + policy.getDataMaskPolicyItems().add(dataMaskPolicyItem); + } else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) { + policy.getRowFilterPolicyItems().add(rowFilterPolicyItem); + } else { // unknown itemType + LOG.warn("RangerPolicyRetriever.getPolicy(policyId=" + policy.getId() + "): ignoring unknown policyItemType " + itemType); + } + } else if (xPolicyItem.getPolicyid().compareTo(policy.getId()) > 0) { + if (iterPolicyItems.hasPrevious()) { + iterPolicyItems.previous(); + } + + break; + } + } + } + } + } +}
http://git-wip-us.apache.org/repos/asf/ranger/blob/d424b1a8/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java new file mode 100644 index 0000000..035e023 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java @@ -0,0 +1,789 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Consolidates Ranger policy details into a JSON string and stores it into a + * column in x_policy table After running this patch Ranger policy can be + * completely read/saved into x_policy table and some related Ref tables (which + * maintain ID->String mapping for each policy). + * + */ + +package org.apache.ranger.patch; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.log4j.Logger; +import org.apache.ranger.authorization.utils.JsonUtils; +import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.biz.TagDBStore; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceResource; +import org.apache.ranger.entity.XXServiceResourceElement; +import org.apache.ranger.entity.XXServiceResourceElementValue; +import org.apache.ranger.entity.XXTag; +import org.apache.ranger.entity.XXTagAttribute; +import org.apache.ranger.entity.XXTagAttributeDef; +import org.apache.ranger.entity.XXTagDef; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.model.RangerValiditySchedule; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.service.RangerServiceResourceService; +import org.apache.ranger.service.RangerTagDefService; +import org.apache.ranger.service.RangerTagService; +import org.apache.ranger.util.CLIUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; + +@Component +public class PatchForUpdatingTagsJson_J10020 extends BaseLoader { + + private static final Logger logger = Logger.getLogger(PatchForUpdatingTagsJson_J10020.class); + + @Autowired + RangerDaoManager daoMgr; + + @Autowired + ServiceDBStore svcStore; + + @Autowired + TagDBStore tagStore; + + @Autowired + @Qualifier(value = "transactionManager") + PlatformTransactionManager txManager; + + @Autowired + RangerTagDefService tagDefService; + + @Autowired + RangerTagService tagService; + + @Autowired + RangerServiceResourceService serviceResourceService; + + public static void main(String[] args) { + logger.info("main()"); + try { + PatchForUpdatingTagsJson_J10020 loader = (PatchForUpdatingTagsJson_J10020) CLIUtil + .getBean(PatchForUpdatingTagsJson_J10020.class); + + loader.init(); + + while (loader.isMoreToProcess()) { + loader.load(); + } + + logger.info("Load complete. Exiting!!!"); + + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + // Do Nothing + } + + @Override + public void execLoad() { + logger.info("==> PatchForUpdatingTagsJson.execLoad()"); + + try { + updateRangerTagsTablesWithTagsJson(); + } catch (Exception e) { + logger.error("Error while UpdateRangerTagsTablesWithTagsJson()", e); + System.exit(1); + } + + logger.info("<== PatchForUpdatingTagsJson.execLoad()"); + } + + @Override + public void printStats() { + logger.info("Update Ranger Tags Tables with Json data "); + } + + private void updateRangerTagsTablesWithTagsJson() throws Exception { + logger.info("==> updateRangerTagsTablesWithTagsJson() "); + + List<RangerService> allServices = svcStore.getServices(new SearchFilter()); + + if (CollectionUtils.isNotEmpty(allServices)) { + for (RangerService service : allServices) { + XXService dbService = daoMgr.getXXService().getById(service.getId()); + RangerTagDBRetriever tagsRetriever = new RangerTagDBRetriever(daoMgr, txManager, dbService); + Map<Long, RangerTagDef> tagDefs = tagsRetriever.getTagDefs(); + Map<Long, RangerTag> tags = tagsRetriever.getTags(); + List<RangerServiceResource> serviceResources = tagsRetriever.getServiceResources(); + + if (MapUtils.isNotEmpty(tagDefs)) { + logger.info("==> Port " + tagDefs.size() + " Tag Definitions for service(name=" + dbService.getName() + ")"); + + for (Map.Entry<Long, RangerTagDef> entry : tagDefs.entrySet()) { + RangerTagDef tagDef = entry.getValue(); + + portTagDef(tagDef); + } + } + + if (MapUtils.isNotEmpty(tags)) { + logger.info("==> Port " + tags.size() + " Tags for service(name=" + dbService.getName() + ")"); + + for (Map.Entry<Long, RangerTag> entry : tags.entrySet()) { + RangerTag tag = entry.getValue(); + + portTag(tag); + } + } + + if (CollectionUtils.isNotEmpty(serviceResources)) { + logger.info("==> Port " + serviceResources.size() + " Service Resources for service(name=" + dbService.getName() + ")"); + + for (RangerServiceResource serviceResource : serviceResources) { + portServiceResource(serviceResource); + } + } + } + } + + logger.info("<== updateRangerTagsTablesWithTagsJson() "); + } + + private void portTagDef(RangerTagDef tagDef) { + tagDefService.update(tagDef); + } + + private void portTag(RangerTag tag) { + tagService.update(tag); + } + + private void portServiceResource(RangerServiceResource serviceResource) throws Exception { + serviceResourceService.update(serviceResource); + tagStore.refreshServiceResource(serviceResource.getId()); + } + + private class RangerTagDBRetriever { + final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class); + + private final RangerDaoManager daoMgr; + private final XXService xService; + private final RangerTagDBRetriever.LookupCache lookupCache; + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + private List<RangerServiceResource> serviceResources; + private Map<Long, RangerTagDef> tagDefs; + private Map<Long, RangerTag> tags; + + RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) throws InterruptedException { + this.daoMgr = daoMgr; + this.xService = xService; + this.lookupCache = new RangerTagDBRetriever.LookupCache(); + this.txManager = txManager; + + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } + + if (this.daoMgr != null && this.xService != null) { + if (this.txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in the same thread and using an existing transaction"); + } + + if (!initializeTagCache(xService)) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in a separate thread and using a new transaction"); + } + + RangerTagDBRetriever.TagLoaderThread t = new RangerTagDBRetriever.TagLoaderThread(txTemplate, xService); + t.setDaemon(true); + t.start(); + t.join(); + } + + } + } + + List<RangerServiceResource> getServiceResources() { + return serviceResources; + } + + Map<Long, RangerTagDef> getTagDefs() { + return tagDefs; + } + + Map<Long, RangerTag> getTags() { + return tags; + } + + private boolean initializeTagCache(XXService xService) { + boolean ret; + RangerTagDBRetriever.TagRetrieverServiceResourceContext serviceResourceContext = new RangerTagDBRetriever.TagRetrieverServiceResourceContext(xService); + RangerTagDBRetriever.TagRetrieverTagDefContext tagDefContext = new RangerTagDBRetriever.TagRetrieverTagDefContext(xService); + RangerTagDBRetriever.TagRetrieverTagContext tagContext = new RangerTagDBRetriever.TagRetrieverTagContext(xService); + + serviceResources = serviceResourceContext.getAllServiceResources(); + tagDefs = tagDefContext.getAllTagDefs(); + tags = tagContext.getAllTags(); + + ret = true; + return ret; + } + + private <T> List<T> asList(T obj) { + List<T> ret = new ArrayList<>(); + + if (obj != null) { + ret.add(obj); + } + + return ret; + } + + private class LookupCache { + final Map<Long, String> userScreenNames = new HashMap<>(); + final Map<Long, String> resourceDefs = new HashMap<>(); + + String getUserScreenName(Long userId) { + String ret = null; + + if (userId != null) { + ret = userScreenNames.get(userId); + + if (ret == null) { + XXPortalUser user = daoMgr.getXXPortalUser().getById(userId); + + if (user != null) { + ret = user.getPublicScreenName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getFirstName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getLoginId(); + } else { + if (!StringUtil.isEmpty(user.getLastName())) { + ret += (" " + user.getLastName()); + } + } + } + + if (ret != null) { + userScreenNames.put(userId, ret); + } + } + } + } + + return ret; + } + + String getResourceName(Long resourceDefId) { + String ret = null; + + if (resourceDefId != null) { + ret = resourceDefs.get(resourceDefId); + + if (ret == null) { + XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); + + if (xResourceDef != null) { + ret = xResourceDef.getName(); + + resourceDefs.put(resourceDefId, ret); + } + } + } + + return ret; + } + } + + private class TagLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + + TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + @Override + public void run() { + txTemplate.setReadOnly(true); + Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() { + @Override + public Boolean doInTransaction(TransactionStatus status) { + boolean ret = initializeTagCache(xService); + + if (!ret) { + status.setRollbackOnly(); + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction"); + } + return ret; + } + }); + + if (LOG.isDebugEnabled()) { + LOG.debug("transaction result:[" + result +"]"); + } + } + } + + private class TagRetrieverServiceResourceContext { + final XXService service; + final ListIterator<XXServiceResource> iterServiceResource; + final ListIterator<XXServiceResourceElement> iterServiceResourceElement; + final ListIterator<XXServiceResourceElementValue> iterServiceResourceElementValue; + + TagRetrieverServiceResourceContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List<XXServiceResource> xServiceResources = daoMgr.getXXServiceResource().findByServiceId(serviceId); + List<XXServiceResourceElement> xServiceResourceElements = daoMgr.getXXServiceResourceElement().findTaggedResourcesInServiceId(serviceId); + List<XXServiceResourceElementValue> xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findTaggedResourcesInServiceId(serviceId); + + this.service = xService; + this.iterServiceResource = xServiceResources.listIterator(); + this.iterServiceResourceElement = xServiceResourceElements.listIterator(); + this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); + + } + + TagRetrieverServiceResourceContext(XXServiceResource xServiceResource, XXService xService) { + Long resourceId = xServiceResource == null ? null : xServiceResource.getId(); + List<XXServiceResource> xServiceResources = asList(xServiceResource); + List<XXServiceResourceElement> xServiceResourceElements = daoMgr.getXXServiceResourceElement().findByResourceId(resourceId); + List<XXServiceResourceElementValue> xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findByResourceId(resourceId); + + this.service = xService; + this.iterServiceResource = xServiceResources.listIterator(); + this.iterServiceResourceElement = xServiceResourceElements.listIterator(); + this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); + } + + List<RangerServiceResource> getAllServiceResources() { + List<RangerServiceResource> ret = new ArrayList<>(); + + while (iterServiceResource.hasNext()) { + RangerServiceResource serviceResource = getNextServiceResource(); + + if (serviceResource != null) { + ret.add(serviceResource); + } + } + + if (!hasProcessedAll()) { + LOG.warn("getAllServiceResources(): perhaps one or more serviceResources got updated during retrieval. Using fallback ... "); + + ret = getServiceResourcesBySecondary(); + } + + return ret; + } + + RangerServiceResource getNextServiceResource() { + RangerServiceResource ret = null; + + if (iterServiceResource.hasNext()) { + XXServiceResource xServiceResource = iterServiceResource.next(); + + if (xServiceResource != null) { + ret = new RangerServiceResource(); + + ret.setId(xServiceResource.getId()); + ret.setGuid(xServiceResource.getGuid()); + ret.setIsEnabled(xServiceResource.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xServiceResource.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xServiceResource.getUpdatedByUserId())); + ret.setCreateTime(xServiceResource.getCreateTime()); + ret.setUpdateTime(xServiceResource.getUpdateTime()); + ret.setVersion(xServiceResource.getVersion()); + ret.setResourceSignature(xServiceResource.getResourceSignature()); + ret.setServiceName(xService.getName()); + + getServiceResourceElements(ret); + } + } + + return ret; + } + + void getServiceResourceElements(RangerServiceResource serviceResource) { + while (iterServiceResourceElement.hasNext()) { + XXServiceResourceElement xServiceResourceElement = iterServiceResourceElement.next(); + + if (xServiceResourceElement.getResourceId().equals(serviceResource.getId())) { + RangerPolicy.RangerPolicyResource resource = new RangerPolicy.RangerPolicyResource(); + + resource.setIsExcludes(xServiceResourceElement.getIsExcludes()); + resource.setIsRecursive(xServiceResourceElement.getIsRecursive()); + + while (iterServiceResourceElementValue.hasNext()) { + XXServiceResourceElementValue xServiceResourceElementValue = iterServiceResourceElementValue.next(); + + if (xServiceResourceElementValue.getResElementId().equals(xServiceResourceElement.getId())) { + resource.getValues().add(xServiceResourceElementValue.getValue()); + } else { + if (iterServiceResourceElementValue.hasPrevious()) { + iterServiceResourceElementValue.previous(); + } + + break; + } + } + + serviceResource.getResourceElements().put(lookupCache.getResourceName(xServiceResourceElement.getResDefId()), resource); + } else if (xServiceResourceElement.getResourceId().compareTo(serviceResource.getId()) > 0) { + if (iterServiceResourceElement.hasPrevious()) { + iterServiceResourceElement.previous(); + } + + break; + } + } + } + + boolean hasProcessedAll() { + boolean moreToProcess = iterServiceResource.hasNext() + || iterServiceResourceElement.hasNext() + || iterServiceResourceElementValue.hasNext(); + + return !moreToProcess; + } + + List<RangerServiceResource> getServiceResourcesBySecondary() { + List<RangerServiceResource> ret = null; + + if (service != null) { + List<XXServiceResource> xServiceResources = daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xServiceResources)) { + ret = new ArrayList<>(xServiceResources.size()); + + for (XXServiceResource xServiceResource : xServiceResources) { + RangerTagDBRetriever.TagRetrieverServiceResourceContext ctx = new RangerTagDBRetriever.TagRetrieverServiceResourceContext(xServiceResource, service); + + RangerServiceResource serviceResource = ctx.getNextServiceResource(); + + if (serviceResource != null) { + ret.add(serviceResource); + } + } + } + } + return ret; + } + } + + private class TagRetrieverTagDefContext { + final XXService service; + final ListIterator<XXTagDef> iterTagDef; + final ListIterator<XXTagAttributeDef> iterTagAttributeDef; + + + TagRetrieverTagDefContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List<XXTagDef> xTagDefs = daoMgr.getXXTagDef().findByServiceId(serviceId); + List<XXTagAttributeDef> xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByServiceId(serviceId); + + this.service = xService; + this.iterTagDef = xTagDefs.listIterator(); + this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); + } + + TagRetrieverTagDefContext(XXTagDef xTagDef, XXService xService) { + Long tagDefId = xTagDef == null ? null : xTagDef.getId(); + List<XXTagDef> xTagDefs = asList(xTagDef); + List<XXTagAttributeDef> xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByTagDefId(tagDefId); + + this.service = xService; + this.iterTagDef = xTagDefs.listIterator(); + this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); + } + + Map<Long, RangerTagDef> getAllTagDefs() { + Map<Long, RangerTagDef> ret = new HashMap<>(); + + while (iterTagDef.hasNext()) { + RangerTagDef tagDef = getNextTagDef(); + + if (tagDef != null) { + ret.put(tagDef.getId(), tagDef); + } + } + + if (!hasProcessedAllTagDefs()) { + LOG.warn("getAllTagDefs(): perhaps one or more tag-definitions got updated during retrieval. Using fallback ... "); + + ret = getTagDefsBySecondary(); + } + + return ret; + } + + RangerTagDef getNextTagDef() { + RangerTagDef ret = null; + + if (iterTagDef.hasNext()) { + XXTagDef xTagDef = iterTagDef.next(); + + if (xTagDef != null) { + ret = new RangerTagDef(); + + ret.setId(xTagDef.getId()); + ret.setGuid(xTagDef.getGuid()); + ret.setIsEnabled(xTagDef.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xTagDef.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xTagDef.getUpdatedByUserId())); + ret.setCreateTime(xTagDef.getCreateTime()); + ret.setUpdateTime(xTagDef.getUpdateTime()); + ret.setVersion(xTagDef.getVersion()); + ret.setName(xTagDef.getName()); + ret.setSource(xTagDef.getSource()); + + getTagAttributeDefs(ret); + } + } + + return ret; + } + + void getTagAttributeDefs(RangerTagDef tagDef) { + while (iterTagAttributeDef.hasNext()) { + XXTagAttributeDef xTagAttributeDef = iterTagAttributeDef.next(); + + if (xTagAttributeDef.getTagDefId().equals(tagDef.getId())) { + RangerTagDef.RangerTagAttributeDef tagAttributeDef = new RangerTagDef.RangerTagAttributeDef(); + + tagAttributeDef.setName(xTagAttributeDef.getName()); + tagAttributeDef.setType(xTagAttributeDef.getType()); + + tagDef.getAttributeDefs().add(tagAttributeDef); + } else if (xTagAttributeDef.getTagDefId().compareTo(tagDef.getId()) > 0) { + if (iterTagAttributeDef.hasPrevious()) { + iterTagAttributeDef.previous(); + } + break; + } + } + } + + boolean hasProcessedAllTagDefs() { + boolean moreToProcess = iterTagAttributeDef.hasNext(); + + return !moreToProcess; + } + + Map<Long, RangerTagDef> getTagDefsBySecondary() { + Map<Long, RangerTagDef> ret = null; + + if (service != null) { + List<XXTagDef> xTagDefs = daoMgr.getXXTagDef().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xTagDefs)) { + ret = new HashMap<>(xTagDefs.size()); + + for (XXTagDef xTagDef : xTagDefs) { + TagRetrieverTagDefContext ctx = new TagRetrieverTagDefContext(xTagDef, service); + + RangerTagDef tagDef = ctx.getNextTagDef(); + + if (tagDef != null) { + ret.put(tagDef.getId(), tagDef); + } + } + } + } + return ret; + } + } + + private class TagRetrieverTagContext { + final XXService service; + final ListIterator<XXTag> iterTag; + final ListIterator<XXTagAttribute> iterTagAttribute; + + + TagRetrieverTagContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + List<XXTag> xTags = daoMgr.getXXTag().findByServiceId(serviceId); + List<XXTagAttribute> xTagAttributes = daoMgr.getXXTagAttribute().findByServiceId(serviceId); + + this.service = xService; + this.iterTag = xTags.listIterator(); + this.iterTagAttribute = xTagAttributes.listIterator(); + + } + + TagRetrieverTagContext(XXTag xTag, XXService xService) { + Long tagId = xTag == null ? null : xTag.getId(); + List<XXTag> xTags = asList(xTag); + List<XXTagAttribute> xTagAttributes = daoMgr.getXXTagAttribute().findByTagId(tagId); + + this.service = xService; + this.iterTag = xTags.listIterator(); + this.iterTagAttribute = xTagAttributes.listIterator(); + } + + + Map<Long, RangerTag> getAllTags() { + Map<Long, RangerTag> ret = new HashMap<>(); + + while (iterTag.hasNext()) { + RangerTag tag = getNextTag(); + + if (tag != null) { + ret.put(tag.getId(), tag); + } + } + + if (!hasProcessedAllTags()) { + LOG.warn("getAllTags(): perhaps one or more tags got updated during retrieval. Using fallback ... "); + + ret = getTagsBySecondary(); + } + + return ret; + } + + RangerTag getNextTag() { + RangerTag ret = null; + + if (iterTag.hasNext()) { + XXTag xTag = iterTag.next(); + + if (xTag != null) { + ret = new RangerTag(); + + ret.setId(xTag.getId()); + ret.setGuid(xTag.getGuid()); + ret.setOwner(xTag.getOwner()); + ret.setCreatedBy(lookupCache.getUserScreenName(xTag.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xTag.getUpdatedByUserId())); + ret.setCreateTime(xTag.getCreateTime()); + ret.setUpdateTime(xTag.getUpdateTime()); + ret.setVersion(xTag.getVersion()); + + Map<String, String> mapOfOptions = JsonUtils.jsonToMapStringString(xTag.getOptions()); + + if (MapUtils.isNotEmpty(mapOfOptions)) { + String validityPeriodsStr = mapOfOptions.get(RangerTag.OPTION_TAG_VALIDITY_PERIODS); + + if (StringUtils.isNotEmpty(validityPeriodsStr)) { + List<RangerValiditySchedule> validityPeriods = JsonUtils.jsonToRangerValiditySchedule(validityPeriodsStr); + + ret.setValidityPeriods(validityPeriods); + } + } + + Map<Long, RangerTagDef> tagDefs = getTagDefs(); + if (tagDefs != null) { + RangerTagDef tagDef = tagDefs.get(xTag.getType()); + if (tagDef != null) { + ret.setType(tagDef.getName()); + } + } + + getTagAttributes(ret); + } + } + + return ret; + } + + void getTagAttributes(RangerTag tag) { + while (iterTagAttribute.hasNext()) { + XXTagAttribute xTagAttribute = iterTagAttribute.next(); + + if (xTagAttribute.getTagId().equals(tag.getId())) { + String attributeName = xTagAttribute.getName(); + String attributeValue = xTagAttribute.getValue(); + + tag.getAttributes().put(attributeName, attributeValue); + } else if (xTagAttribute.getTagId().compareTo(tag.getId()) > 0) { + if (iterTagAttribute.hasPrevious()) { + iterTagAttribute.previous(); + } + break; + } + } + } + + boolean hasProcessedAllTags() { + boolean moreToProcess = iterTagAttribute.hasNext(); + + return !moreToProcess; + } + + Map<Long, RangerTag> getTagsBySecondary() { + Map<Long, RangerTag> ret = null; + + if (service != null) { + List<XXTag> xTags = daoMgr.getXXTag().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xTags)) { + ret = new HashMap<>(xTags.size()); + + for (XXTag xTag : xTags) { + TagRetrieverTagContext ctx = new TagRetrieverTagContext(xTag, service); + + RangerTag tag = ctx.getNextTag(); + + if (tag != null) { + ret.put(tag.getId(), tag); + } + } + } + } + return ret; + } + } + } +} + http://git-wip-us.apache.org/repos/asf/ranger/blob/d424b1a8/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 97b8d71..c513548 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -101,8 +101,8 @@ import org.apache.ranger.plugin.policyengine.RangerPolicyEngineCacheForEngineOpt import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.service.ResourceLookupContext; -import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; @@ -1515,10 +1515,11 @@ public class ServiceREST { String serviceName = request.getParameter(PARAM_SERVICE_NAME); String policyName = request.getParameter(PARAM_POLICY_NAME); String updateIfExists = request.getParameter(PARAM_UPDATE_IF_EXISTS); - - if (serviceName == null && policyName == null && updateIfExists != null && updateIfExists.equalsIgnoreCase("true")){ - serviceName = (String) request.getAttribute(PARAM_SERVICE_NAME); - policyName = (String) request.getAttribute(PARAM_POLICY_NAME); + + if (serviceName == null && policyName == null && updateIfExists != null + && updateIfExists.equalsIgnoreCase("true")) { + serviceName = (String) request.getAttribute(PARAM_SERVICE_NAME); + policyName = (String) request.getAttribute(PARAM_POLICY_NAME); } if(StringUtils.isNotEmpty(serviceName)) { @@ -1529,7 +1530,7 @@ public class ServiceREST { policy.setName(StringUtils.trim(policyName)); } - if(updateIfExists != null && Boolean.valueOf(updateIfExists)) { + if (updateIfExists != null && Boolean.valueOf(updateIfExists)) { RangerPolicy existingPolicy = null; try { if(StringUtils.isNotEmpty(policy.getGuid())) { @@ -2413,7 +2414,7 @@ public class ServiceREST { validator.validate(rangerPolicy.getId(), Action.DELETE); ensureAdminAccess(rangerPolicy); bizUtil.blockAuditorRoleUser(); - svcStore.deletePolicy(rangerPolicy); + svcStore.deletePolicy(rangerPolicy.getId()); totalDeletedPilicies = totalDeletedPilicies + 1; if (LOG.isDebugEnabled()) { LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully." ); @@ -3149,7 +3150,7 @@ public class ServiceREST { ret.addAll(listToFilter); } } else if (isServiceAdminUser) { - ret.addAll(listToFilter); + ret.addAll(listToFilter); } continue; @@ -3552,4 +3553,4 @@ public class ServiceREST { } } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ranger/blob/d424b1a8/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java b/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java index e6a519a..fe1883d 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java @@ -23,7 +23,7 @@ import org.apache.ranger.entity.XXDBBase; import org.springframework.stereotype.Component; @Component -public class RangerAuditFields { +public class RangerAuditFields<T extends XXDBBase> { public <T extends XXDBBase, PARENT extends XXDBBase> T populateAuditFields(T xObj, PARENT parentObj) { xObj.setCreateTime(parentObj.getCreateTime()); http://git-wip-us.apache.org/repos/asf/ranger/blob/d424b1a8/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java index 8d42165..6ab12ad 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java @@ -17,7 +17,6 @@ package org.apache.ranger.service; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.common.GUIDUtil; @@ -30,12 +29,10 @@ import org.apache.ranger.common.SortField.SORT_ORDER; import org.apache.ranger.entity.XXPolicyBase; import org.apache.ranger.entity.XXService; import org.apache.ranger.plugin.model.RangerPolicy; -import org.apache.ranger.plugin.model.RangerValiditySchedule; import org.apache.ranger.plugin.util.SearchFilter; import org.springframework.beans.factory.annotation.Autowired; import java.util.HashMap; -import java.util.List; import java.util.Map; public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends RangerPolicy> extends @@ -58,18 +55,20 @@ public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends "XXService xSvc", "xSvc.id = obj.service")); searchFields .add(new SearchField(SearchFilter.IS_ENABLED, "obj.isEnabled", DATA_TYPE.BOOLEAN, SEARCH_TYPE.FULL)); - searchFields.add(new SearchField(SearchFilter.IS_RECURSIVE,"xPolRes.isRecursive",DATA_TYPE.BOOLEAN,SEARCH_TYPE.FULL, - "XXPolicyResource xPolRes","obj.id=xPolRes.policyId")); + //might need updation + /*searchFields.add(new SearchField(SearchFilter.IS_RECURSIVE,"xPolRes.isRecursive",DATA_TYPE.BOOLEAN,SEARCH_TYPE.FULL, + "XXPolicyResource xPolRes","obj.id=xPolRes.policyId"));*/ searchFields.add(new SearchField(SearchFilter.POLICY_ID, "obj.id", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.POLICY_NAME, "obj.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.GUID, "obj.guid", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.USER, "xUser.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL, - "XXUser xUser, XXPolicyItem xPolItem, XXPolicyItemUserPerm userPerm", "obj.id = xPolItem.policyId " - + "and userPerm.policyItemId = xPolItem.id and xUser.id = userPerm.userId")); + "XXUser xUser, XXPolicyRefUser refUser", "obj.id = refUser.policyId " + + "and xUser.id = refUser.userId")); searchFields.add(new SearchField(SearchFilter.GROUP, "xGrp.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL, - "XXGroup xGrp, XXPolicyItem xPolItem, XXPolicyItemGroupPerm grpPerm", "obj.id = xPolItem.policyId " - + "and grpPerm.policyItemId = xPolItem.id and xGrp.id = grpPerm.groupId")); - searchFields.add(new SearchField(SearchFilter.POL_RESOURCE, "resMap.value", DATA_TYPE.STRING, + "XXGroup xGrp , XXPolicyRefGroup refGroup", "obj.id = refGroup.policyId " + + "and xGrp.id = refGroup.groupId")); + //might need updation + /*searchFields.add(new SearchField(SearchFilter.POL_RESOURCE, "resMap.value", DATA_TYPE.STRING, SEARCH_TYPE.PARTIAL, "XXPolicyResourceMap resMap, XXPolicyResource polRes", "resMap.resourceId = polRes.id and polRes.policyId = obj.id")); /*searchFields.add(new SearchField(SearchFilter.POLICY_LABELS_PARTIAL, "obj.label_name", DATA_TYPE.STRING, @@ -118,9 +117,11 @@ public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends options.remove(OPTION_POLICY_VALIDITY_SCHEDULES); } - xObj.setOptions(JsonUtils.mapToJson(options)); + xObj.setOptions(JsonUtils.mapToJson(options)); - return xObj; + xObj.setPolicyText(JsonUtils.objectToJson(vObj)); + + return xObj; } @Override @@ -137,20 +138,16 @@ public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends vObj.setIsEnabled(xObj.getIsEnabled()); vObj.setIsAuditEnabled(xObj.getIsAuditEnabled()); - Map<String, Object> options = JsonUtils.jsonToObject(xObj.getOptions(), Map.class); - - if (MapUtils.isNotEmpty(options)) { - String optionPolicyValiditySchedule = (String)options.remove(OPTION_POLICY_VALIDITY_SCHEDULES); + String policyText = xObj.getPolicyText(); - if (StringUtils.isNotBlank(optionPolicyValiditySchedule)) { - List<RangerValiditySchedule> validitySchedules = JsonUtils.jsonToRangerValiditySchedule(optionPolicyValiditySchedule); + RangerPolicy ret = JsonUtils.jsonToObject(policyText, RangerPolicy.class); - vObj.setValiditySchedules(validitySchedules); - } + if (ret != null) { + vObj.setOptions(ret.getOptions()); + vObj.setValiditySchedules(ret.getValiditySchedules()); + vObj.setPolicyLabels(ret.getPolicyLabels()); } - vObj.setOptions(options); - return vObj; } } http://git-wip-us.apache.org/repos/asf/ranger/blob/d424b1a8/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefServiceBase.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefServiceBase.java index 5cbe47a..9b543ef 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefServiceBase.java @@ -62,7 +62,7 @@ public abstract class RangerServiceDefServiceBase<T extends XXServiceDefBase, V private static final String OPTION_RESOURCE_IS_VALID_LEAF = "__isValidLeaf"; @Autowired - RangerAuditFields rangerAuditFields; + RangerAuditFields<?> rangerAuditFields; @Autowired JSONUtil jsonUtil;
