This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.5
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.5 by this push:
new 03925dbc6 RANGER-4852: De-duplicated tags do not work correctly when
tag-deltas are enabled
03925dbc6 is described below
commit 03925dbc646aa85a6a81f94a39327ce1f7bd414b
Author: Abhay Kulkarni <akulka...@cloudera.com>
AuthorDate: Fri Jul 12 16:13:39 2024 -0700
RANGER-4852: De-duplicated tags do not work correctly when tag-deltas are
enabled
(cherry picked from commit 9648f8f8c9e6f178837942ee76a992636e83f772)
---
.../contextenricher/RangerAdminTagRetriever.java | 13 ---
.../RangerFileBasedTagRetriever.java | 8 +-
.../plugin/contextenricher/RangerTagEnricher.java | 7 +-
.../plugin/contextenricher/RangerTagRetriever.java | 1 +
.../ranger/plugin/model/RangerServiceTags.java | 2 +-
.../ranger/plugin/util/RangerCommonConstants.java | 4 +-
.../plugin/util/RangerServiceTagsDeltaUtil.java | 124 ++++++++++++---------
.../org/apache/ranger/plugin/util/ServiceTags.java | 57 ++++++++--
.../java/org/apache/ranger/biz/TagDBStore.java | 18 ++-
.../ranger/common/RangerServiceTagsCache.java | 4 +-
10 files changed, 148 insertions(+), 90 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
index b2b7d5f71..bae31e6cc 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
@@ -33,12 +33,7 @@ import java.util.Map;
public class RangerAdminTagRetriever extends RangerTagRetriever {
private static final Logger LOG =
LoggerFactory.getLogger(RangerAdminTagRetriever.class);
- private static final String OPTION_DEDUP_TAGS = "deDupTags";
- private static final Boolean OPTION_DEDUP_TAGS_DEFAULT = true;
-
-
private RangerAdminClient adminClient;
- private boolean deDupTags;
@Override
public void init(Map<String, String> options) {
@@ -50,11 +45,9 @@ public class RangerAdminTagRetriever extends
RangerTagRetriever {
pluginConfig = new
RangerPluginConfig(serviceDef.getName(), serviceName, appId, null, null, null);
}
- String deDupTagsVal = options != null?
options.get(OPTION_DEDUP_TAGS) : null;
RangerPluginContext pluginContext = getPluginContext();
RangerAdminClient rangerAdmin =
pluginContext.getAdminClient();
- this.deDupTags = StringUtils.isNotBlank(deDupTagsVal)
? Boolean.parseBoolean(deDupTagsVal) : OPTION_DEDUP_TAGS_DEFAULT;
this.adminClient = (rangerAdmin != null) ? rangerAdmin
: pluginContext.createAdminClient(pluginConfig);
} else {
LOG.error("FATAL: Cannot find service/serviceDef to use
for retrieving tags. Will NOT be able to retrieve tags.");
@@ -78,12 +71,6 @@ public class RangerAdminTagRetriever extends
RangerTagRetriever {
}
}
- if (serviceTags != null && !serviceTags.getIsDelta() &&
deDupTags) {
- final int countOfDuplicateTags =
serviceTags.dedupTags();
-
- LOG.info("Number of duplicate tags removed from the
received serviceTags:[" + countOfDuplicateTags + "]. Number of tags in the
de-duplicated serviceTags :[" + serviceTags.getTags().size() + "].");
- }
-
return serviceTags;
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
index 2a3643399..1e39d93fa 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
@@ -37,7 +37,6 @@ public class RangerFileBasedTagRetriever extends
RangerTagRetriever {
private URL serviceTagsFileURL;
private String serviceTagsFileName;
- private boolean deDupTags;
int tagFilesCount = 0;
int currentTagFileIndex = 0;
boolean isInitial = true;
@@ -50,15 +49,12 @@ public class RangerFileBasedTagRetriever extends
RangerTagRetriever {
String serviceTagsFileNameProperty = "serviceTagsFileName";
String serviceTagsDefaultFileName =
"/testdata/test_servicetags_hive.json";
- String deDupTagsProperty = "deDupTags";
String tagFilesCountProperty = "tagFileCount";
if (StringUtils.isNotBlank(serviceName) && serviceDef != null
&& StringUtils.isNotBlank(appId)) {
// Open specified file from options- it should contain
service-tags
serviceTagsFileName = options != null?
options.get(serviceTagsFileNameProperty) : null;
- String deDupTagsVal = options != null?
options.get(deDupTagsProperty) : "false";
- deDupTags =
Boolean.parseBoolean(deDupTagsVal);
serviceTagsFileName = serviceTagsFileName == null ?
serviceTagsDefaultFileName : serviceTagsFileName;
if (options != null) {
@@ -179,7 +175,7 @@ public class RangerFileBasedTagRetriever extends
RangerTagRetriever {
) {
ret = JsonUtils.jsonToObject(reader,
ServiceTags.class);
- if (deDupTags) {
+ if (ret.getIsTagsDeduped()) {
final int countOfDuplicateTags
= ret.dedupTags();
LOG.info("Number of duplicate
tags removed from the received serviceTags:[" + countOfDuplicateTags + "].
Number of tags in the de-duplicated serviceTags :[" + ret.getTags().size() +
"].");
}
@@ -204,7 +200,7 @@ public class RangerFileBasedTagRetriever extends
RangerTagRetriever {
ret = JsonUtils.jsonToObject(reader,
ServiceTags.class);
currentTagFileIndex++;
- if (deDupTags) {
+ if (ret.getIsTagsDeduped()) {
final int countOfDuplicateTags
= ret.dedupTags();
LOG.info("Number of duplicate
tags removed from the received serviceTags:[" + countOfDuplicateTags + "].
Number of tags in the de-duplicated serviceTags :[" + ret.getTags().size() +
"].");
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index 0208e6892..2fa24eba6 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -309,13 +309,18 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
}
if (!serviceTags.getIsDelta()) {
+ if (serviceTags.getIsTagsDeduped()) {
+ final int countOfDuplicateTags
= serviceTags.dedupTags();
+
+ LOG.info("Number of duplicate
tags removed from the received serviceTags:[" + countOfDuplicateTags + "].
Number of tags in the de-duplicated serviceTags :[" +
serviceTags.getTags().size() + "].");
+ }
processServiceTags(serviceTags);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Received service-tag
deltas:" + serviceTags);
}
ServiceTags oldServiceTags =
enrichedServiceTags != null ? enrichedServiceTags.getServiceTags() : new
ServiceTags();
- ServiceTags allServiceTags =
rebuildOnlyIndex ? oldServiceTags :
RangerServiceTagsDeltaUtil.applyDelta(oldServiceTags, serviceTags);
+ ServiceTags allServiceTags =
rebuildOnlyIndex ? oldServiceTags :
RangerServiceTagsDeltaUtil.applyDelta(oldServiceTags, serviceTags,
serviceTags.getIsTagsDeduped());
if (serviceTags.getTagsChangeExtent()
== ServiceTags.TagsChangeExtent.NONE) {
if (LOG.isDebugEnabled()) {
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
index d7c737525..584959e02 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
@@ -71,4 +71,5 @@ public abstract class RangerTagRetriever {
public void setPluginContext(RangerPluginContext pluginContext) {
this.pluginContext = pluginContext;
}
+
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceTags.java
b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceTags.java
index 59288f7d5..3a2633974 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceTags.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceTags.java
@@ -170,7 +170,7 @@ public class RangerServiceTags implements
java.io.Serializable {
ret = new ServiceTags(toServiceTagsOp(tags.getOp()),
tags.getServiceName(),
tags.tagVersion, tags.getTagUpdateTime(),
tags.getTagDefinitions(), tags.getTags(),
tags.getServiceResources(),
tags.getResourceToTagIds(), false,
- ServiceTags.TagsChangeExtent.ALL);
+ ServiceTags.TagsChangeExtent.ALL, false);
}
return ret;
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java
index 23689790d..9d6e1f0b5 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java
@@ -40,7 +40,7 @@ public class RangerCommonConstants {
public static final String RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES
= ".supports.in.place.tag.updates";
public static final String PLUGIN_CONFIG_SUFFIX_IN_PLACE_TAG_UPDATES
= ".supports.in.place.tag.updates";
- public static final String RANGER_ADMIN_SUPPORTS_TAGS_DEDUP
= ".supports.tags.dedup";
+ public static final String RANGER_SUPPORTS_TAGS_DEDUP
= ".supports.tags.dedup";
public static final boolean RANGER_ADMIN_SUFFIX_POLICY_DELTA_DEFAULT
= false;
public static final boolean PLUGIN_CONFIG_SUFFIX_POLICY_DELTA_DEFAULT
= false;
@@ -54,7 +54,7 @@ public class RangerCommonConstants {
public static final boolean
RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT = false;
public static final boolean
PLUGIN_CONFIG_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT = false;
- public static final boolean RANGER_ADMIN_SUPPORTS_TAGS_DEDUP_DEFAULT
= true;
+ public static final boolean RANGER_SUPPORTS_TAGS_DEDUP_DEFAULT
= true;
public static final boolean POLICY_REST_CLIENT_SESSION_COOKIE_ENABLED
= true;
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java
index f2e68aed9..d5a0915e0 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java
@@ -19,8 +19,10 @@
package org.apache.ranger.plugin.util;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.ranger.authorization.hadoop.config.RangerAdminConfig;
+import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
import org.apache.ranger.plugin.model.RangerTagDef;
@@ -42,16 +44,13 @@ public class RangerServiceTagsDeltaUtil {
private static final Logger PERF_TAGS_DELTA_LOG =
RangerPerfTracer.getPerfLogger("tags.delta");
- private static boolean SUPPORTS_TAGS_DEDUP_INITIALIZED = false;
- private static boolean SUPPORTS_TAGS_DEDUP = false;
-
/*
It should be possible to call applyDelta() multiple times with serviceTags
and delta resulting from previous call to applyDelta()
The end result should be same if called once or multiple times.
*/
- static public ServiceTags applyDelta(ServiceTags serviceTags, ServiceTags
delta) {
+ static public ServiceTags applyDelta(ServiceTags serviceTags, ServiceTags
delta, boolean supportsTagsDedup) {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> RangerServiceTagsDeltaUtil.applyDelta()");
+ LOG.debug("==> RangerServiceTagsDeltaUtil.applyDelta():
serviceTags:[" + serviceTags + "], delta:[" + delta + "], supportsTagsDedup:["
+ supportsTagsDedup + "]");
}
ServiceTags ret = serviceTags;
@@ -66,6 +65,7 @@ public class RangerServiceTagsDeltaUtil {
ret.setServiceName(delta.getServiceName());
ret.setTagVersion(delta.getTagVersion());
+ ret.setIsTagsDeduped(delta.getIsTagsDeduped());
int tagDefsAdded = 0, tagDefsUpdated = 0, tagDefsRemoved = 0;
int tagsAdded = 0, tagsUpdated = 0, tagsRemoved = 0;
@@ -103,25 +103,47 @@ public class RangerServiceTagsDeltaUtil {
RangerTag deltaTag = entry.getValue();
if (StringUtils.isEmpty(deltaTag.getType())) { // tag has been
removed
- RangerTag removedTag = tags.remove(deltaTagId);
-
- if (removedTag != null) {
- tagsRemoved++;
+ if (supportsTagsDedup) {
+ boolean found = false;
+
+ for (Iterator<Map.Entry<RangerTag, MutablePair<Long,
Long>>> iterator = ret.cachedTags.entrySet().iterator(); iterator.hasNext(); ) {
+ MutablePair<Long, Long> value =
iterator.next().getValue();
+ if (value.left.equals(deltaTagId)) {
+ if (--value.right == 0) {
+ // This may never be true when this tag is
duplicated
+ // as the mapping between de-duplicated
tags is not maintained - only the reference count is stored
+ // So, the tag with the smallest tag-id
(among duplicate tags) will never be removed
+ if (tags.remove(deltaTagId) != null) {
+ tagsRemoved++;
+ }
+ iterator.remove();
+ }
+ found = true;
+ break;
+ }
+ }
- if (isSupportsTagsDedup()) {
- ret.cachedTags.remove(removedTag);
+ if (!found) {
+ if (tags.remove(deltaTagId) != null) {
+ tagsRemoved++;
+ }
+ }
+ } else {
+ if (tags.remove(deltaTagId) != null) {
+ tagsRemoved++;
}
}
} else {
- if (isSupportsTagsDedup()) {
- Long cachedTagId = ret.cachedTags.get(deltaTag);
+ if (supportsTagsDedup) {
+ MutablePair<Long, Long> cachedTag =
ret.cachedTags.get(deltaTag);
- if (cachedTagId == null) {
- ret.cachedTags.put(deltaTag, deltaTagId);
+ if (cachedTag == null) {
+ ret.cachedTags.put(deltaTag, new
MutablePair<>(deltaTagId, 1L));
tags.put(deltaTagId, deltaTag);
+ tagsAdded++;
} else {
- replacedTagIds.put(deltaTagId, cachedTagId);
- deltaTagIter.remove();
+ cachedTag.right++;
+ replacedTagIds.put(deltaTagId, cachedTag.left);
}
} else {
RangerTag existing = tags.put(deltaTagId, deltaTag);
@@ -147,43 +169,29 @@ public class RangerServiceTagsDeltaUtil {
if (existingResource != null) {
if
(StringUtils.isNotEmpty(resource.getResourceSignature())) {
if
(!StringUtils.equals(resource.getResourceSignature(),
existingResource.getResourceSignature())) { // ServiceResource changed;
replace existing instance
- existingResource.setResourceSignature(null);
-
- boolean isAddedResource =
resourcesToAdd.remove(existingResource.getId()) == existingResource;
-
- // if a resource with this ID was already removed,
don't replace the entry in resourcesToRemove
- if (!isAddedResource &&
!resourcesToRemove.containsKey(existingResource.getId())) {
-
resourcesToRemove.put(existingResource.getId(), existingResource);
- }
-
+ /* If the signature changed, we need to remove
existing resource and add new resource */
+ resourcesToRemove.put(resource.getId(),
existingResource);
resourcesToAdd.put(resource.getId(), resource);
- idResourceMap.put(resource.getId(), resource);
}
} else { // resource deleted
- boolean isAddedResource =
resourcesToAdd.remove(existingResource.getId()) == existingResource;
-
- if (!isAddedResource) {
- resourcesToRemove.put(existingResource.getId(),
existingResource);
- }
+ resourcesToRemove.put(resource.getId(),
existingResource);
- idResourceMap.remove(existingResource.getId());
resourceToTagIds.remove(existingResource.getId());
}
} else { // resource added
if
(StringUtils.isNotEmpty(resource.getResourceSignature())) {
resourcesToAdd.put(resource.getId(), resource);
-
- idResourceMap.put(resource.getId(), resource);
}
}
}
if (!resourcesToRemove.isEmpty()) {
for (ListIterator<RangerServiceResource> iter =
serviceResources.listIterator(); iter.hasNext(); ) {
- RangerServiceResource resource = iter.next();
- RangerServiceResource replacedResource =
resourcesToRemove.get(resource.getId());
+ RangerServiceResource resource = iter.next();
+ RangerServiceResource deletedResource =
resourcesToRemove.get(resource.getId());
+ RangerServiceResource addedResource =
resourcesToAdd.get(resource.getId());
- if (replacedResource == resource) {
+ if (addedResource == null && deletedResource == resource) {
iter.remove();
}
}
@@ -208,16 +216,33 @@ public class RangerServiceTagsDeltaUtil {
resourceToTagIds.putAll(delta.getResourceToTagIds());
+ if (MapUtils.isEmpty(resourceToTagIds)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("There are no resource->tag mappings!!");
+ }
+ if (MapUtils.isNotEmpty(ret.getTags())) {
+ LOG.warn("There are no resource->tag mappings, but there
are tags in the ServiceTags!! Cleaning up");
+ ret.getTags().clear();
+ }
+ if (supportsTagsDedup) {
+ ret.cachedTags.clear();
+ }
+ }
+
// Ensure that any modified service-resources are at head of list
of service-resources in delta
// So that in setServiceTags(), they get cleaned out first, and
service-resource with new spec gets added
- if (!resourcesToAdd.isEmpty()) {
- List<RangerServiceResource> deltaServiceResources = new
ArrayList<>(resourcesToAdd.values());
- deltaServiceResources.addAll(delta.getServiceResources());
-
- delta.setServiceResources(deltaServiceResources);
+ List<RangerServiceResource> deltaServiceResources = new
ArrayList<>();
+ for (RangerServiceResource resourceToRemove :
resourcesToRemove.values()) {
+ resourceToRemove.setResourceSignature(null);
+ deltaServiceResources.add(resourceToRemove);
+ }
+ if (!resourcesToAdd.isEmpty()) {
+ deltaServiceResources.addAll(resourcesToAdd.values());
}
+ delta.setServiceResources(deltaServiceResources);
+
if (LOG.isDebugEnabled()) {
LOG.debug("RangerServiceTagsDeltaUtil.applyDelta():
delta(tagDefs={}, tags={}, resources={}), " +
"resources(total={}, added={}, removed={}), " +
@@ -235,7 +260,7 @@ public class RangerServiceTagsDeltaUtil {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== RangerServiceTagsDeltaUtil.applyDelta()");
+ LOG.debug("<== RangerServiceTagsDeltaUtil.applyDelta():
serviceTags:[" + ret + "], delta:[" + delta + "], supportsTagsDedup:[" +
supportsTagsDedup + "]");
}
RangerPerfTracer.log(perf);
@@ -312,13 +337,4 @@ public class RangerServiceTagsDeltaUtil {
}
}
- public static boolean isSupportsTagsDedup() {
- if (!SUPPORTS_TAGS_DEDUP_INITIALIZED) {
- RangerAdminConfig config = RangerAdminConfig.getInstance();
-
- SUPPORTS_TAGS_DEDUP = config.getBoolean("ranger.admin" +
RangerCommonConstants.RANGER_ADMIN_SUPPORTS_TAGS_DEDUP,
RangerCommonConstants.RANGER_ADMIN_SUPPORTS_TAGS_DEDUP_DEFAULT);
- SUPPORTS_TAGS_DEDUP_INITIALIZED = true;
- }
- return SUPPORTS_TAGS_DEDUP;
- }
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
index 39110357d..cc2ebe53a 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.HashMap;
import java.util.ArrayList;
+import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.ranger.authorization.utils.StringUtil;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
@@ -61,9 +62,11 @@ public class ServiceTags implements java.io.Serializable {
private Map<Long, List<Long>> resourceToTagIds;
private Boolean isDelta;
private TagsChangeExtent tagsChangeExtent;
+ private Boolean isTagsDeduped;
+ // MutablePair.left is the tag-id, MutablePair.right is the
reference-count
@JsonIgnore
- Map<RangerTag, Long> cachedTags = new HashMap<>();
+ Map<RangerTag, MutablePair<Long, Long>> cachedTags = new HashMap<>();
public ServiceTags() {
this(OP_ADD_OR_UPDATE, null, 0L, null, null, null, null, null);
@@ -71,10 +74,10 @@ public class ServiceTags implements java.io.Serializable {
public ServiceTags(String op, String serviceName, Long tagVersion, Date
tagUpdateTime, Map<Long, RangerTagDef> tagDefinitions,
Map<Long, RangerTag> tags,
List<RangerServiceResource> serviceResources, Map<Long, List<Long>>
resourceToTagIds) {
- this(op, serviceName, tagVersion, tagUpdateTime,
tagDefinitions, tags, serviceResources, resourceToTagIds, false,
TagsChangeExtent.ALL);
+ this(op, serviceName, tagVersion, tagUpdateTime,
tagDefinitions, tags, serviceResources, resourceToTagIds, false,
TagsChangeExtent.ALL, false);
}
public ServiceTags(String op, String serviceName, Long tagVersion, Date
tagUpdateTime, Map<Long, RangerTagDef> tagDefinitions,
- Map<Long, RangerTag> tags,
List<RangerServiceResource> serviceResources, Map<Long, List<Long>>
resourceToTagIds, Boolean isDelta, TagsChangeExtent tagsChangeExtent) {
+ Map<Long, RangerTag> tags,
List<RangerServiceResource> serviceResources, Map<Long, List<Long>>
resourceToTagIds, Boolean isDelta, TagsChangeExtent tagsChangeExtent, Boolean
isTagsDeduped) {
setOp(op);
setServiceName(serviceName);
setTagVersion(tagVersion);
@@ -85,6 +88,7 @@ public class ServiceTags implements java.io.Serializable {
setResourceToTagIds(resourceToTagIds);
setIsDelta(isDelta);
setTagsChangeExtent(tagsChangeExtent);
+ setIsTagsDeduped(isTagsDeduped);
}
public ServiceTags(ServiceTags other) {
@@ -97,6 +101,7 @@ public class ServiceTags implements java.io.Serializable {
setServiceResources(other.getServiceResources() != null ? new
ArrayList<>(other.getServiceResources()) : null);
setResourceToTagIds(other.getResourceToTagIds() != null ? new
HashMap<>(other.getResourceToTagIds()) : null);
setIsDelta(other.getIsDelta());
+ setIsTagsDeduped(other.getIsTagsDeduped());
setTagsChangeExtent(other.getTagsChangeExtent());
this.cachedTags = new HashMap<>(other.cachedTags);
@@ -198,6 +203,14 @@ public class ServiceTags implements java.io.Serializable {
this.isDelta = isDelta;
}
+ public Boolean getIsTagsDeduped() {
+ return isTagsDeduped == null ? Boolean.FALSE : isTagsDeduped;
+ }
+
+ public void setIsTagsDeduped(Boolean isTagsDeduped) {
+ this.isTagsDeduped = isTagsDeduped;
+ }
+
public TagsChangeExtent getTagsChangeExtent() {
return tagsChangeExtent;
}
@@ -225,6 +238,8 @@ public class ServiceTags implements java.io.Serializable {
.append(",
serviceResources={").append(serviceResources).append("}")
.append(", tags={").append(tags).append("}")
.append(",
resourceToTagIds={").append(resourceToTagIds).append("}")
+ .append(",
isTagsDeduped={").append(isTagsDeduped).append("}")
+ .append(",
cachedTags={").append(cachedTags).append("}")
.append("}");
return sb;
@@ -234,31 +249,51 @@ public class ServiceTags implements java.io.Serializable {
final int ret;
final Map<Long, Long> replacedIds = new HashMap<>();
final int initialTagsCount = tags.size();
+ final List<Long> tagIdsToRemove = new ArrayList<>();
for (Iterator<Map.Entry<Long, RangerTag>> iter =
tags.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry<Long, RangerTag> entry = iter.next();
Long tagId = entry.getKey();
RangerTag tag =
entry.getValue();
- Long cachedTagId =
cachedTags.get(tag);
+ MutablePair<Long, Long> cachedTag =
cachedTags.get(tag);
- if (cachedTagId == null) {
- cachedTags.put(tag, tagId);
+ if (cachedTag == null) {
+ cachedTags.put(tag, new MutablePair<>(tagId,
0L)); // reference count will be incremented later
} else {
- replacedIds.put(tagId, cachedTagId);
- iter.remove();
+ if (tagId < cachedTag.left) {
+ replacedIds.put(cachedTag.left, tagId);
+ tagIdsToRemove.add(cachedTag.left);
+ cachedTag.left = tagId;
+ } else {
+ replacedIds.put(tagId, cachedTag.left);
+ iter.remove();
+ }
}
}
+ for (Long tagIdToRemove : tagIdsToRemove) {
+ tags.remove(tagIdToRemove);
+ }
final int finalTagsCount = tags.size();
for (Map.Entry<Long, List<Long>> resourceEntry :
resourceToTagIds.entrySet()) {
for (ListIterator<Long> listIter =
resourceEntry.getValue().listIterator(); listIter.hasNext(); ) {
Long tagId = listIter.next();
- Long replacerTagId = replacedIds.get(tagId);
- if (replacerTagId != null) {
- listIter.set(replacerTagId);
+ for (Long replacerTagId =
replacedIds.get(tagId); replacerTagId != null; replacerTagId =
replacedIds.get(tagId)) {
+ tagId = replacerTagId;
}
+
+ listIter.set(tagId);
+
+ RangerTag tag = tags.get(tagId);
+ if (tag != null) { // This should always
be true
+ MutablePair<Long, Long> cachedTag =
cachedTags.get(tag);
+ if (cachedTag != null) { // This should
always be true
+ cachedTag.right++;
+ }
+ }
+
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index 9ecbb14ac..9134e3988 100755
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -1033,7 +1033,9 @@ public class TagDBStore extends AbstractTagStore {
ret.setServiceResources(resources);
ret.setResourceToTagIds(resourceToTagIds);
- if (RangerServiceTagsDeltaUtil.isSupportsTagsDedup()) {
+ ret.setIsTagsDeduped(isSupportsTagsDedup());
+
+ if (isSupportsTagsDedup()) {
final int countOfDuplicateTags =
ret.dedupTags();
if (LOG.isDebugEnabled()) {
LOG.debug("Number of duplicate tags
removed from the received serviceTags:[" + countOfDuplicateTags + "]. Number of
tags in the de-duplicated serviceTags :[" + ret.getTags().size() + "].");
@@ -1243,6 +1245,7 @@ public class TagDBStore extends AbstractTagStore {
if (CollectionUtils.isNotEmpty(serviceResourceIds) ||
CollectionUtils.isNotEmpty(tagIds)) {
ret = new ServiceTags();
ret.setIsDelta(true);
+ ret.setIsTagsDeduped(isSupportsTagsDedup());
ServiceTags.TagsChangeExtent tagsChangeExtent =
ServiceTags.TagsChangeExtent.TAGS;
@@ -1457,4 +1460,17 @@ public class TagDBStore extends AbstractTagStore {
return ret;
}
+
+ private static boolean SUPPORTS_TAGS_DEDUP_INITIALIZED = false;
+ private static boolean SUPPORTS_TAGS_DEDUP = false;
+
+ public static boolean isSupportsTagsDedup() {
+ if (!SUPPORTS_TAGS_DEDUP_INITIALIZED) {
+ RangerAdminConfig config =
RangerAdminConfig.getInstance();
+
+ SUPPORTS_TAGS_DEDUP = config.getBoolean("ranger.admin"
+ RangerCommonConstants.RANGER_SUPPORTS_TAGS_DEDUP,
RangerCommonConstants.RANGER_SUPPORTS_TAGS_DEDUP_DEFAULT);
+ SUPPORTS_TAGS_DEDUP_INITIALIZED = true;
+ }
+ return SUPPORTS_TAGS_DEDUP;
+ }
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java
b/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java
index 2aecc4388..ec0ff7083 100644
---
a/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java
+++
b/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java
@@ -22,6 +22,7 @@ package org.apache.ranger.common;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.ranger.authorization.hadoop.config.RangerAdminConfig;
+import org.apache.ranger.biz.TagDBStore;
import org.apache.ranger.plugin.store.TagStore;
import org.apache.ranger.plugin.util.RangerServiceTagsDeltaUtil;
@@ -376,7 +377,8 @@ public class RangerServiceTagsCache {
LOG.debug("Retrieved
tag-deltas from database. These will be applied on top of ServiceTags
version:[" + cachedServiceTagsVersion + "], tag-deltas:[" +
serviceTagsFromDb.getTagVersion() + "]");
}
- this.serviceTags =
RangerServiceTagsDeltaUtil.applyDelta(serviceTags, serviceTagsFromDb);
+ boolean supportsTagsDedeup =
TagDBStore.isSupportsTagsDedup();
+ this.serviceTags =
RangerServiceTagsDeltaUtil.applyDelta(serviceTags, serviceTagsFromDb,
supportsTagsDedeup);
this.deltaCache = new
ServiceTagsDeltasCache(cachedServiceTagsVersion, serviceTagsFromDb);
}
} else {
