This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 13341c7 RANGER-2510: Support for Incremental tag updates to improve
performance - handle updates to tag policies
13341c7 is described below
commit 13341c7c22ca78eff2d1eb49bb5b56e8cb68f8d3
Author: Abhay Kulkarni <ab...@apache.org>
AuthorDate: Sun Oct 6 20:31:26 2019 -0700
RANGER-2510: Support for Incremental tag updates to improve performance -
handle updates to tag policies
---
.../policyengine/RangerPolicyEngineCache.java | 1 +
.../policyengine/RangerPolicyEngineImpl.java | 28 ++++++++--
.../policyengine/RangerPolicyRepository.java | 64 +---------------------
.../ranger/plugin/service/RangerAuthContext.java | 22 ++++++--
.../ranger/plugin/service/RangerBasePlugin.java | 16 ++++--
.../ranger/plugin/util/RangerPolicyDeltaUtil.java | 2 +-
.../ranger/plugin/util/RangerRESTClient.java | 2 +-
.../apache/ranger/plugin/util/ServiceDefUtil.java | 64 ++++++++++++++++++++++
.../apache/ranger/plugin/util/ServicePolicies.java | 8 +--
.../java/org/apache/ranger/biz/ServiceDBStore.java | 2 +-
10 files changed, 124 insertions(+), 85 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java
index 5dae0c1..251a0ec 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java
@@ -103,6 +103,7 @@ public class RangerPolicyEngineCache {
}
} else {
ret = addPolicyEngine(policies, options);
+
((RangerPolicyEngineImpl)policyEngine).setIsShared(false);
}
return ret;
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
index 576d5e5..b81f50c 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
@@ -85,9 +85,9 @@ public class RangerPolicyEngineImpl implements
RangerPolicyEngine {
private Map<String, RangerPolicyRepository> policyRepositories = new
HashMap<>();
private Map<String, RangerResourceTrie> trieMap;
- private Map<String, Set<String>> userRoleMapping;
- private Map<String, Set<String>> groupRoleMapping;
private Map<String, String> zoneTagServiceMap;
+ private Map<String, Set<String>> userRoleMapping;
+ private Map<String, Set<String>> groupRoleMapping;
private final RangerPluginContext pluginContext;
public RangerPolicyEngineImpl(final RangerPolicyEngineImpl other,
ServicePolicies servicePolicies) {
@@ -96,7 +96,7 @@ public class RangerPolicyEngineImpl implements
RangerPolicyEngine {
public RangerPolicyEngineImpl(final RangerPolicyEngineImpl other,
ServicePolicies servicePolicies, RangerRoles rangerRoles) {
- long policyVersion =
servicePolicies.getPolicyVersion();
+ long policyVersion = servicePolicies.getPolicyVersion();
this.useForwardedIPAddress = other.useForwardedIPAddress;
this.trustedProxyAddresses = other.trustedProxyAddresses;
@@ -178,8 +178,13 @@ public class RangerPolicyEngineImpl implements
RangerPolicyEngine {
}
}
- if (other.policyRepository != null &&
CollectionUtils.isNotEmpty(defaultZoneDeltas)) {
- this.policyRepository = new
RangerPolicyRepository(other.policyRepository, defaultZoneDeltas,
policyVersion);
+ if (CollectionUtils.isNotEmpty(defaultZoneDeltas)) {
+ if (other.policyRepository == null) {
+ LOG.warn("Current policy-engine's
policy-repository is null! Should not have happened!!");
+ this.policyRepository = other.policyRepository;
+ } else {
+ this.policyRepository = new
RangerPolicyRepository(other.policyRepository, defaultZoneDeltas,
policyVersion);
+ }
} else {
this.policyRepository =
shareWith(other.policyRepository);
}
@@ -221,6 +226,9 @@ public class RangerPolicyEngineImpl implements
RangerPolicyEngine {
reorderPolicyEvaluators();
+ RangerAuthContext oldContext = pluginContext.getAuthContext();
+ this.pluginContext.setAuthContext(new RangerAuthContext(this,
oldContext));
+
}
public RangerPolicyEngineImpl(String appId, ServicePolicies
servicePolicies, RangerPolicyEngineOptions options) {
@@ -390,6 +398,16 @@ public class RangerPolicyEngineImpl implements
RangerPolicyEngine {
return ret;
}
+ public void setIsShared(boolean isShared) {
+ this.policyRepository.setIsShared(isShared);
+ if (this.tagPolicyRepository != null) {
+ this.tagPolicyRepository.setIsShared(isShared);
+ }
+ for (RangerPolicyRepository repository :
policyRepositories.values()) {
+ repository.setIsShared(isShared);
+ }
+ }
+
@Override
protected void finalize() throws Throwable {
try {
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index ae88c73..abc57df 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -337,7 +337,7 @@ class RangerPolicyRepository {
this.zoneName = null;
- this.serviceDef =
normalizeAccessTypeDefs(ServiceDefUtil.normalize(tagPolicies.getServiceDef()),
componentServiceDef.getName());
+ this.serviceDef =
ServiceDefUtil.normalizeAccessTypeDefs(ServiceDefUtil.normalize(tagPolicies.getServiceDef()),
componentServiceDef.getName());
this.componentServiceDef = componentServiceDef;
this.appId = appId;
@@ -706,68 +706,6 @@ class RangerPolicyRepository {
return ret;
}
- private RangerServiceDef normalizeAccessTypeDefs(RangerServiceDef
serviceDef, final String componentType) {
-
- if (serviceDef != null && StringUtils.isNotBlank(componentType)) {
-
- List<RangerServiceDef.RangerAccessTypeDef> accessTypeDefs =
serviceDef.getAccessTypes();
-
- if (CollectionUtils.isNotEmpty(accessTypeDefs)) {
-
- String prefix = componentType +
AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR;
-
- List<RangerServiceDef.RangerAccessTypeDef>
unneededAccessTypeDefs = null;
-
- for (RangerServiceDef.RangerAccessTypeDef accessTypeDef :
accessTypeDefs) {
-
- String accessType = accessTypeDef.getName();
-
- if (StringUtils.startsWith(accessType, prefix)) {
-
- String newAccessType =
StringUtils.removeStart(accessType, prefix);
-
- accessTypeDef.setName(newAccessType);
-
- Collection<String> impliedGrants =
accessTypeDef.getImpliedGrants();
-
- if (CollectionUtils.isNotEmpty(impliedGrants)) {
-
- Collection<String> newImpliedGrants = null;
-
- for (String impliedGrant : impliedGrants) {
-
- if (StringUtils.startsWith(impliedGrant,
prefix)) {
-
- String newImpliedGrant =
StringUtils.removeStart(impliedGrant, prefix);
-
- if (newImpliedGrants == null) {
- newImpliedGrants = new ArrayList<>();
- }
-
- newImpliedGrants.add(newImpliedGrant);
- }
- }
- accessTypeDef.setImpliedGrants(newImpliedGrants);
-
- }
- } else if (StringUtils.contains(accessType,
AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR)) {
- if(unneededAccessTypeDefs == null) {
- unneededAccessTypeDefs = new ArrayList<>();
- }
-
- unneededAccessTypeDefs.add(accessTypeDef);
- }
- }
-
- if(unneededAccessTypeDefs != null) {
- accessTypeDefs.removeAll(unneededAccessTypeDefs);
- }
- }
- }
-
- return serviceDef;
- }
-
private List<RangerPolicy> normalizeAndPrunePolicies(List<RangerPolicy>
rangerPolicies, final String componentType) {
if (CollectionUtils.isNotEmpty(rangerPolicies) &&
StringUtils.isNotBlank(componentType)) {
List<RangerPolicy> policiesToPrune = null;
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerAuthContext.java
b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerAuthContext.java
index 6cd1df6..6611104 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerAuthContext.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerAuthContext.java
@@ -49,8 +49,8 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
public class RangerAuthContext implements RangerPolicyEngine {
- private static final Log LOG =
LogFactory.getLog(RangerAuthContext.class);
- private final RangerPluginContext rangerPluginContext;
+ private static final Log LOG = LogFactory.getLog(RangerAuthContext.class);
+ private final RangerPluginContext rangerPluginContext;
private final RangerPolicyEngine policyEngine;
private final Map<RangerContextEnricher, Object> requestContextEnrichers;
@@ -63,15 +63,13 @@ public class RangerAuthContext implements
RangerPolicyEngine {
RangerAuthContext(RangerAuthContext other) {
if (other != null) {
this.policyEngine = other.getPolicyEngine();
-
+ this.rangerPluginContext = other.rangerPluginContext;
Map<RangerContextEnricher, Object> localReference =
other.requestContextEnrichers;
if (MapUtils.isNotEmpty(localReference)) {
this.requestContextEnrichers = new
ConcurrentHashMap<>(localReference);
} else {
this.requestContextEnrichers = new ConcurrentHashMap<>();
}
-
- this.rangerPluginContext = other.rangerPluginContext;
} else {
this.policyEngine = null;
this.requestContextEnrichers = new ConcurrentHashMap<>();
@@ -79,6 +77,20 @@ public class RangerAuthContext implements RangerPolicyEngine
{
}
}
+ public RangerAuthContext(RangerPolicyEngine policyEngine,
RangerAuthContext other) {
+ this.policyEngine = policyEngine;
+
+ if (other != null) {
+ Map<RangerContextEnricher, Object> localReference =
other.requestContextEnrichers;
+
+ this.rangerPluginContext = other.rangerPluginContext;
+ this.requestContextEnrichers = MapUtils.isNotEmpty(localReference)
? new ConcurrentHashMap<>(localReference) : new ConcurrentHashMap<>();
+ } else {
+ this.rangerPluginContext = null;
+ this.requestContextEnrichers = new ConcurrentHashMap<>();
+ }
+ }
+
public RangerPolicyEngine getPolicyEngine() {
return policyEngine;
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
index 1325a40..df09b15 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -325,7 +325,8 @@ public class RangerBasePlugin {
}
if (isValid) {
- RangerPolicyEngine newPolicyEngine = null;
+ RangerPolicyEngine newPolicyEngine = null;
+ boolean isPolicyEngineShared = false;
if(updateRangerRolesOnly) {
this.policyEngine.setRangerRoles(rangerRoles);
@@ -347,6 +348,7 @@ public class RangerBasePlugin {
if
(LOG.isDebugEnabled()) {
LOG.debug("Applied policyDeltas=" +
Arrays.toString(policies.getPolicyDeltas().toArray()) + ")");
}
+ isPolicyEngineShared =
true;
} else {
if
(LOG.isDebugEnabled()) {
LOG.debug("Failed to apply policyDeltas=" +
Arrays.toString(policies.getPolicyDeltas().toArray()) + "), Creating engine
from policies");
@@ -363,15 +365,19 @@ public class RangerBasePlugin {
if (newPolicyEngine != null) {
-
newPolicyEngine.setUseForwardedIPAddress(useForwardedIPAddress);
-
newPolicyEngine.setTrustedProxyAddresses(trustedProxyAddresses);
+ if (!isPolicyEngineShared) {
+
newPolicyEngine.setUseForwardedIPAddress(useForwardedIPAddress);
+
newPolicyEngine.setTrustedProxyAddresses(trustedProxyAddresses);
+ }
+
this.policyEngine = newPolicyEngine;
this.currentAuthContext = new
RangerAuthContext(rangerPluginContext.getAuthContext());
contextChanged();
- if (oldPolicyEngine != null &&
!oldPolicyEngine.preCleanup()) {
- LOG.error("preCleanup() failed
on the previous policy engine instance !!");
+ if (oldPolicyEngine != null &&
!isPolicyEngineShared) {
+
((RangerPolicyEngineImpl)oldPolicyEngine).setIsShared(false);
+ oldPolicyEngine.preCleanup();
}
if (this.refresher != null) {
this.refresher.saveToCache(usePolicyDeltas ? servicePolicies : policies);
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
index 9c50f8a..4599997 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
@@ -59,7 +59,7 @@ public class RangerPolicyDeltaUtil {
for (RangerPolicyDelta delta : deltas) {
int changeType = delta.getChangeType();
if (!serviceType.equals(delta.getServiceType())) {
- if
(!delta.getServiceType().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME))
{
+ if
(!serviceType.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME) &&
!delta.getServiceType().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME))
{
LOG.error("Found unexpected serviceType in
policyDelta:[" + delta + "]. Was expecting serviceType:[" + serviceType + "].
Should NOT have come here!! Ignoring delta and continuing");
}
continue;
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java
index ef5f1d5..7d30b89 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java
@@ -121,7 +121,7 @@ public class RangerRESTClient {
mUrl = url;
mSslConfigFileName = sslConfigFileName;
this.configuredURLs = getURLs(mUrl);
- this.lastKnownActiveUrlIndex = (new
Random()).nextInt(configuredURLs.size());
+ this.lastKnownActiveUrlIndex = configuredURLs.size() == 0 ? 0 :
(new Random()).nextInt(configuredURLs.size());
init();
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
index 596f5e8..f383241 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
@@ -28,9 +28,11 @@ import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
+import org.apache.ranger.plugin.store.AbstractServiceStore;
import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.HashMap;
import java.util.Map;
@@ -164,6 +166,68 @@ public class ServiceDefUtil {
return StringUtils.isEmpty(val) ? defaultValue : val.charAt(0);
}
+ public static RangerServiceDef normalizeAccessTypeDefs(RangerServiceDef
serviceDef, final String componentType) {
+
+ if (serviceDef != null && StringUtils.isNotBlank(componentType)) {
+
+ List<RangerServiceDef.RangerAccessTypeDef> accessTypeDefs =
serviceDef.getAccessTypes();
+
+ if (CollectionUtils.isNotEmpty(accessTypeDefs)) {
+
+ String prefix = componentType +
AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR;
+
+ List<RangerServiceDef.RangerAccessTypeDef>
unneededAccessTypeDefs = null;
+
+ for (RangerServiceDef.RangerAccessTypeDef accessTypeDef :
accessTypeDefs) {
+
+ String accessType = accessTypeDef.getName();
+
+ if (StringUtils.startsWith(accessType, prefix)) {
+
+ String newAccessType =
StringUtils.removeStart(accessType, prefix);
+
+ accessTypeDef.setName(newAccessType);
+
+ Collection<String> impliedGrants =
accessTypeDef.getImpliedGrants();
+
+ if (CollectionUtils.isNotEmpty(impliedGrants)) {
+
+ Collection<String> newImpliedGrants = null;
+
+ for (String impliedGrant : impliedGrants) {
+
+ if (StringUtils.startsWith(impliedGrant,
prefix)) {
+
+ String newImpliedGrant =
StringUtils.removeStart(impliedGrant, prefix);
+
+ if (newImpliedGrants == null) {
+ newImpliedGrants = new ArrayList<>();
+ }
+
+ newImpliedGrants.add(newImpliedGrant);
+ }
+ }
+ accessTypeDef.setImpliedGrants(newImpliedGrants);
+
+ }
+ } else if (StringUtils.contains(accessType,
AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR)) {
+ if(unneededAccessTypeDefs == null) {
+ unneededAccessTypeDefs = new ArrayList<>();
+ }
+
+ unneededAccessTypeDefs.add(accessTypeDef);
+ }
+ }
+
+ if(unneededAccessTypeDefs != null) {
+ accessTypeDefs.removeAll(unneededAccessTypeDefs);
+ }
+ }
+ }
+
+ return serviceDef;
+ }
+
private static void normalizeDataMaskDef(RangerServiceDef serviceDef) {
if(serviceDef != null && serviceDef.getDataMaskDef() != null) {
List<RangerResourceDef> dataMaskResources =
serviceDef.getDataMaskDef().getResources();
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java
index f6beac6..360404a 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java
@@ -355,21 +355,21 @@ public class ServicePolicies implements
java.io.Serializable {
ret.setPolicies(Collections.emptyList());
ret.setPolicyDeltas(null);
if (source.getTagPolicies() != null) {
- TagPolicies tagPolicies =
copyHeader(source.getTagPolicies());
+ TagPolicies tagPolicies =
copyHeader(source.getTagPolicies(), source.getServiceDef().getName());
ret.setTagPolicies(tagPolicies);
}
return ret;
}
- static public TagPolicies copyHeader(TagPolicies source) {
+ static public TagPolicies copyHeader(TagPolicies source, String
componentServiceName) {
TagPolicies ret = new TagPolicies();
ret.setServiceName(source.getServiceName());
ret.setServiceId(source.getServiceId());
ret.setPolicyVersion(source.getPolicyVersion());
ret.setAuditMode(source.getAuditMode());
- ret.setServiceDef(source.getServiceDef());
+
ret.setServiceDef(ServiceDefUtil.normalizeAccessTypeDefs(source.getServiceDef(),
componentServiceName));
ret.setPolicyUpdateTime(source.getPolicyUpdateTime());
ret.setPolicies(Collections.emptyList());
@@ -390,7 +390,7 @@ public class ServicePolicies implements
java.io.Serializable {
if (servicePolicies.getTagPolicies() != null) {
newTagPolicies =
RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies,
servicePolicies.getPolicyDeltas(),
servicePolicies.getTagPolicies().getServiceDef().getName());
} else {
- newTagPolicies = null;
+ newTagPolicies = oldTagPolicies;
}
if (ret.getTagPolicies() != null) {
diff --git
a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index 51e08e1..edc886c 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -3012,7 +3012,7 @@ public class ServiceDBStore extends AbstractServiceStore {
ret.setPolicies(null);
ret.setPolicyDeltas(compressedDeltas);
- if
(CollectionUtils.isNotEmpty(tagPolicyDeltas)) {
+ if (tagServiceDef != null &&
tagService != null) {
ServicePolicies.TagPolicies tagPolicies = new ServicePolicies.TagPolicies();
tagPolicies.setServiceDef(tagServiceDef);
tagPolicies.setServiceId(tagService.getId());
