This is an automated email from the ASF dual-hosted git repository. pradeep pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push: new c5bf2f6 RANGER-2528: Export API to get zone, unzone as well as tag based policies from Ranger. c5bf2f6 is described below commit c5bf2f6364a97539451656d28fd36e35d8e2736d Author: Sanjar Matin <sanjarmati...@gmail.com> AuthorDate: Sat Sep 21 18:10:14 2019 +0530 RANGER-2528: Export API to get zone, unzone as well as tag based policies from Ranger. Signed-off-by: Pradeep <prad...@apache.org> --- .../apache/ranger/plugin/util/SearchFilter.java | 2 + .../java/org/apache/ranger/biz/ServiceDBStore.java | 129 ++++++++++++++++----- .../org/apache/ranger/common/RangerSearchUtil.java | 2 + .../java/org/apache/ranger/common/ServiceUtil.java | 42 +++---- .../java/org/apache/ranger/rest/ServiceREST.java | 15 ++- 5 files changed, 134 insertions(+), 56 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java index 029b104..93b28a8 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java @@ -86,6 +86,8 @@ public class SearchFilter { public static final String PLUGIN_ENTITY_TYPE = "pluginEntityType"; public static final String PLUGIN_IP_ADDRESS = "pluginIpAddress"; public static final String CLUSTER_NAME = "clusterName"; + public static final String FETCH_ZONE_UNZONE_POLICIES = "fetchZoneAndUnzonePolicies"; + public static final String FETCH_TAG_POLICIES = "fetchTagPolicies"; private Map<String, String> params; private int startIndex; diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index fc4b40d..e1c4578 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -19,6 +19,13 @@ package org.apache.ranger.biz; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.UnknownHostException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -26,6 +33,7 @@ import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -33,13 +41,6 @@ import java.util.Map.Entry; import java.util.Set; import java.util.StringTokenizer; import java.util.TreeSet; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.UnknownHostException; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import javax.annotation.PostConstruct; import javax.servlet.ServletOutputStream; @@ -62,27 +63,12 @@ import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.ContextUtil; -import org.apache.ranger.common.MessageEnums; -import org.apache.ranger.common.RangerCommonEnums; -import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter; -import org.apache.ranger.db.XXPolicyDao; -import org.apache.ranger.entity.*; -import org.apache.ranger.plugin.model.RangerRole; -import org.apache.ranger.plugin.model.RangerSecurityZone; -import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; -import org.apache.ranger.plugin.model.validation.RangerValidator; -import org.apache.ranger.plugin.model.validation.ValidationFailureDetails; -import org.apache.ranger.plugin.model.RangerPolicyDelta; -import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; -import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; -import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; -import org.apache.ranger.plugin.service.RangerBaseService; -import org.apache.ranger.plugin.store.ServiceStore; -import org.apache.ranger.plugin.util.PasswordUtils; import org.apache.ranger.common.DateUtil; import org.apache.ranger.common.JSONUtil; +import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerCommonEnums; import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.RangerFactory; import org.apache.ranger.common.RangerServicePoliciesCache; @@ -90,6 +76,7 @@ import org.apache.ranger.common.RangerVersionInfo; import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.StringUtil; import org.apache.ranger.common.UserSessionBase; +import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.db.XXAccessTypeDefDao; import org.apache.ranger.db.XXAccessTypeDefGrantsDao; @@ -98,12 +85,39 @@ import org.apache.ranger.db.XXDataMaskTypeDefDao; import org.apache.ranger.db.XXEnumDefDao; import org.apache.ranger.db.XXEnumElementDefDao; import org.apache.ranger.db.XXPolicyConditionDefDao; +import org.apache.ranger.db.XXPolicyDao; import org.apache.ranger.db.XXPolicyLabelMapDao; import org.apache.ranger.db.XXResourceDefDao; import org.apache.ranger.db.XXServiceConfigDefDao; import org.apache.ranger.db.XXServiceConfigMapDao; import org.apache.ranger.db.XXServiceDao; import org.apache.ranger.db.XXServiceVersionInfoDao; +import org.apache.ranger.entity.XXAccessTypeDef; +import org.apache.ranger.entity.XXAccessTypeDefGrants; +import org.apache.ranger.entity.XXContextEnricherDef; +import org.apache.ranger.entity.XXDataHist; +import org.apache.ranger.entity.XXDataMaskTypeDef; +import org.apache.ranger.entity.XXEnumDef; +import org.apache.ranger.entity.XXEnumElementDef; +import org.apache.ranger.entity.XXGroup; +import org.apache.ranger.entity.XXPolicy; +import org.apache.ranger.entity.XXPolicyChangeLog; +import org.apache.ranger.entity.XXPolicyConditionDef; +import org.apache.ranger.entity.XXPolicyLabel; +import org.apache.ranger.entity.XXPolicyLabelMap; +import org.apache.ranger.entity.XXPolicyRefAccessType; +import org.apache.ranger.entity.XXPolicyRefCondition; +import org.apache.ranger.entity.XXPolicyRefResource; +import org.apache.ranger.entity.XXResourceDef; +import org.apache.ranger.entity.XXRoleRefRole; +import org.apache.ranger.entity.XXSecurityZone; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceConfigDef; +import org.apache.ranger.entity.XXServiceConfigMap; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.entity.XXServiceVersionInfo; +import org.apache.ranger.entity.XXTrxLog; +import org.apache.ranger.entity.XXUser; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; @@ -111,7 +125,10 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicyDelta; import org.apache.ranger.plugin.model.RangerPolicyResourceSignature; +import org.apache.ranger.plugin.model.RangerRole; +import org.apache.ranger.plugin.model.RangerSecurityZone; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef; @@ -125,10 +142,19 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerRowFilterDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; +import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; +import org.apache.ranger.plugin.model.validation.RangerValidator; +import org.apache.ranger.plugin.model.validation.ValidationFailureDetails; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; +import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.ranger.plugin.service.RangerBaseService; import org.apache.ranger.plugin.store.AbstractServiceStore; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.ServicePredicateUtil; +import org.apache.ranger.plugin.store.ServiceStore; +import org.apache.ranger.plugin.util.PasswordUtils; import org.apache.ranger.plugin.util.RangerPolicyDeltaUtil; import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; @@ -1810,7 +1836,6 @@ public class ServiceDBStore extends AbstractServiceStore { MessageEnums.OPER_NO_PERMISSION); } } - return xService == null ? null : svcService.getPopulatedViewObject(xService); } @@ -2155,14 +2180,60 @@ public class ServiceDBStore extends AbstractServiceStore { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.getPolicies()"); } + Boolean fetchTagPolicies = Boolean.valueOf(filter.getParam(SearchFilter.FETCH_TAG_POLICIES)); + Boolean fetchAllZonePolicies = Boolean.valueOf(filter.getParam(SearchFilter.FETCH_ZONE_UNZONE_POLICIES)); + String zoneName = filter.getParam(SearchFilter.ZONE_NAME); + + List<RangerPolicy> ret = new ArrayList<RangerPolicy>(); RangerPolicyList policyList = searchRangerPolicies(filter); - List<RangerPolicy> ret = policyList.getPolicies(); + List<RangerPolicy> resourcePolicies = policyList.getPolicies(); + List<RangerPolicy> tagPolicies = new ArrayList<RangerPolicy>(); + + if(fetchTagPolicies) { + tagPolicies = searchRangerTagPoliciesOnBasisOfServiceName(resourcePolicies); + Iterator<RangerPolicy> itr = tagPolicies.iterator(); + while (itr.hasNext()) { + RangerPolicy pol = (RangerPolicy) itr.next(); + if(!fetchAllZonePolicies) { + if(StringUtils.isNotEmpty(zoneName)) { + if(!zoneName.equals(pol.getZoneName())){ + itr.remove(); + } + } else { + if(StringUtils.isNotEmpty(pol.getZoneName())) { + itr.remove(); + } + } + } + } + } if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceDBStore.getPolicies()"); } + ret.addAll(resourcePolicies); + ret.addAll(tagPolicies); return ret; } + private List<RangerPolicy> searchRangerTagPoliciesOnBasisOfServiceName(List<RangerPolicy> allExceptTagPolicies) throws Exception { + Set<String> rangerServiceNames = new HashSet<String>(); + for(RangerPolicy pol : allExceptTagPolicies) { + rangerServiceNames.add(pol.getService()); + } + List<RangerPolicy> retPolicies = new ArrayList<RangerPolicy>(); + for(String eachRangerService : rangerServiceNames) { + List<RangerPolicy> policies = new ArrayList<RangerPolicy>(); + RangerService rangerServiceObj = getServiceByName(eachRangerService); + RangerService rangerTagService = getServiceByName(rangerServiceObj.getTagService()); + if(rangerTagService != null) { + ServicePolicies servicePolicies = RangerServicePoliciesCache.getInstance().getServicePolicies(rangerTagService.getName(),rangerTagService.getId(), -1L, true, this); + policies = servicePolicies != null ? servicePolicies.getPolicies() : null; + retPolicies.addAll(policies); + } + } + return retPolicies; + } + @Override public Long getPolicyId(final Long serviceId, final String policyName, final Long zoneId) { if(LOG.isDebugEnabled()) { @@ -2264,8 +2335,10 @@ public class ServiceDBStore extends AbstractServiceStore { } List<RangerPolicy> ret = getServicePolicies(service, filter); - if(StringUtils.isBlank(zoneName)) { - ret = noZoneFilter(ret); + if(!"true".equalsIgnoreCase(filter.getParam(SearchFilter.FETCH_ZONE_UNZONE_POLICIES))) { + if(StringUtils.isBlank(zoneName)) { + ret = noZoneFilter(ret); + } } if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceDBStore.getServicePolicies(" + serviceId + ") : policy-count=" + (ret == null ? 0 : ret.size())); diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java index 954144b..99af818 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java @@ -81,6 +81,8 @@ public class RangerSearchUtil extends SearchUtil { ret.setParam(SearchFilter.GROUP_NAME_PARTIAL, request.getParameter(SearchFilter.GROUP_NAME_PARTIAL)); ret.setParam(SearchFilter.USER_NAME_PARTIAL, request.getParameter(SearchFilter.USER_NAME_PARTIAL)); ret.setParam(SearchFilter.CLUSTER_NAME, request.getParameter(SearchFilter.CLUSTER_NAME)); + ret.setParam(SearchFilter.FETCH_ZONE_UNZONE_POLICIES, request.getParameter(SearchFilter.FETCH_ZONE_UNZONE_POLICIES)); + ret.setParam(SearchFilter.FETCH_TAG_POLICIES, request.getParameter(SearchFilter.FETCH_TAG_POLICIES)); for (Map.Entry<String, String[]> e : request.getParameterMap().entrySet()) { String name = e.getKey(); String[] values = e.getValue(); diff --git a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java index 7c44e6c..2b1a3fa 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java @@ -1578,6 +1578,7 @@ public class ServiceUtil { public List<RangerPolicy> getMatchingPoliciesForResource(HttpServletRequest request, List<RangerPolicy> policyLists) { List<RangerPolicy> policies = new ArrayList<RangerPolicy>(); + final String serviceTypeForTag = EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME; if (request != null) { String resource = request.getParameter(SearchFilter.POL_RESOURCE); String serviceType = request.getParameter(SearchFilter.SERVICE_TYPE); @@ -1587,32 +1588,25 @@ public class ServiceUtil { RangerPolicy.RangerPolicyResource rangerPolicyResource = null; for (RangerPolicy rangerPolicy : policyLists) { if (rangerPolicy != null) { - rangerPolicyResourceMap = rangerPolicy.getResources(); - if (rangerPolicyResourceMap != null) { - if (rangerPolicyResourceMap.containsKey("path")) { - rangerPolicyResource = rangerPolicyResourceMap.get("path"); - if (rangerPolicyResource != null) { - resourceList = rangerPolicyResource.getValues(); - if (CollectionUtils.isNotEmpty(resourceList) && resourceList.size() == 1) { - String resourcePath = resourceList.get(0); - if (!StringUtil.isEmpty(resourcePath)) { - if (resourcePath.equals(resource) - || resourcePath.startsWith(resource + "/")) { - policies.add(rangerPolicy); - } + if(serviceTypeForTag.equals(rangerPolicy.getServiceType())) { + policies.add(rangerPolicy); + }else { + rangerPolicyResourceMap = rangerPolicy.getResources(); + if (rangerPolicyResourceMap != null) { + if (rangerPolicyResourceMap.containsKey("path")) { + rangerPolicyResource = rangerPolicyResourceMap.get("path"); + if (rangerPolicyResource != null) { + resourceList = rangerPolicyResource.getValues(); + if (CollectionUtils.isNotEmpty(resourceList) && resourceList.contains(resource)) { + policies.add(rangerPolicy); } } - } - } else if (rangerPolicyResourceMap.containsKey("database")) { - rangerPolicyResource = rangerPolicyResourceMap.get("database"); - if (rangerPolicyResource != null) { - resourceList = rangerPolicyResource.getValues(); - if (CollectionUtils.isNotEmpty(resourceList) && resourceList.size() == 1) { - String resourcePath = resourceList.get(0); - if (!StringUtil.isEmpty(resourcePath)) { - if (resourcePath.equals(resource)) { - policies.add(rangerPolicy); - } + } else if (rangerPolicyResourceMap.containsKey("database")) { + rangerPolicyResource = rangerPolicyResourceMap.get("database"); + if (rangerPolicyResource != null) { + resourceList = rangerPolicyResource.getValues(); + if (CollectionUtils.isNotEmpty(resourceList) && resourceList.contains(resource)) { + policies.add(rangerPolicy); } } } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 3d44315..8ee181a 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -2050,7 +2050,7 @@ public class ServiceREST { RangerPerfTracer perf = null; SearchFilter filter = searchUtil.getSearchFilter(request,policyService.sortFields); - String zoneName = filter.getParam("zoneName"); + requestParamsValidation(filter); try { if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG,"ServiceREST.getPoliciesInJson()"); @@ -2062,9 +2062,7 @@ public class ServiceREST { List<RangerPolicy> policyLists = new ArrayList<RangerPolicy>(); policyLists = getAllFilteredPolicyList(filter, request, policyLists); - if (StringUtils.isBlank(zoneName)) {// if zoneName not provided in search parameter, return only policies which are not in any zone. - policyLists = svcStore.noZoneFilter(policyLists); - } + if (CollectionUtils.isNotEmpty(policyLists)) { for (RangerPolicy rangerPolicy : policyLists) { if (rangerPolicy != null) { @@ -2102,6 +2100,15 @@ public class ServiceREST { } } + private void requestParamsValidation(SearchFilter filter) { + Boolean fetchAllZonePolicies = Boolean.valueOf(filter.getParam(SearchFilter.FETCH_ZONE_UNZONE_POLICIES)); + String zoneName = filter.getParam(SearchFilter.ZONE_NAME); + + if (fetchAllZonePolicies && StringUtils.isNotEmpty(zoneName)) { + throw restErrorUtil.createRESTException("Invalid parameter: " + SearchFilter.ZONE_NAME + " can not be provided, along with " + SearchFilter.FETCH_ZONE_UNZONE_POLICIES + "=true"); + } + } + @POST @Path("/policies/importPoliciesFromFile") @Consumes({MediaType.MULTIPART_FORM_DATA, MediaType.APPLICATION_JSON})