This is an automated email from the ASF dual-hosted git repository. abhay pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push: new e76101d28 RANGER-4379: Assorted debugging help : save policy-cache at Ranger-admin and policy-cache as well as downloaded policy-deltas on plugin side e76101d28 is described below commit e76101d28b001217f81ffcbd0647714a07fe68c3 Author: Abhay Kulkarni <ab...@apache.org> AuthorDate: Mon Sep 25 07:59:44 2023 -0700 RANGER-4379: Assorted debugging help : save policy-cache at Ranger-admin and policy-cache as well as downloaded policy-deltas on plugin side --- .../plugin/policyengine/RangerResourceTrie.java | 12 ++-- .../ranger/plugin/service/RangerBasePlugin.java | 7 ++- .../apache/ranger/plugin/util/PolicyRefresher.java | 66 +++++++++++++++++++--- .../ranger/common/RangerServicePoliciesCache.java | 44 +++++++++++++++ 4 files changed, 116 insertions(+), 13 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java index 647059203..2f725036d 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java @@ -1305,14 +1305,14 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> { void toString(StringBuilder sb) { String nodeValue = this.str; - sb.append("nodeValue=").append(nodeValue); + sb.append("nodeValue=").append(nodeValue == null ? "ROOT" : nodeValue); sb.append("; isSetup=").append(isSetup); sb.append("; isSharingParentWildcardEvaluators=").append(isSharingParentWildcardEvaluators); sb.append("; childCount=").append(children.size()); - sb.append("; evaluators=[ "); + sb.append("; evaluators=["); if (evaluators != null) { for (U evaluator : evaluators) { - sb.append(evaluator.getId()).append(" "); + sb.append(evaluator.getId()).append(","); } } sb.append("]"); @@ -1320,7 +1320,7 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> { sb.append("; wildcardEvaluators=[ "); if (wildcardEvaluators != null) { for (U evaluator : wildcardEvaluators) { - sb.append(evaluator.getId()).append(" "); + sb.append(evaluator.getId()).append(","); } } sb.append("]"); @@ -1329,6 +1329,10 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> { void toString(String prefix, StringBuilder sb) { String nodeValue = prefix + (str != null ? str : ""); + if (!nodeValue.equals(prefix)) { + prefix = prefix + "|"; + } + sb.append(prefix); toString(sb); sb.append("]\n"); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index f1eb08e4e..2f4af9763 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -410,7 +410,9 @@ public class RangerBasePlugin { newPolicyEngine.setTrustedProxyAddresses(pluginConfig.getTrustedProxyAddresses()); } + LOG.info("Switching policy engine from [" + getPolicyVersion() + "]"); this.policyEngine = newPolicyEngine; + LOG.info("Switched policy engine to [" + getPolicyVersion() + "]"); this.currentAuthContext = pluginContext.getAuthContext(); pluginContext.notifyAuthContextChanged(); @@ -516,7 +518,6 @@ public class RangerBasePlugin { if (resultProcessor != null) { resultProcessor.processResult(ret); } - return ret; } @@ -1327,4 +1328,8 @@ public class RangerBasePlugin { return ret; } + + public Long getPolicyVersion() { + return this.policyEngine == null ? -1L : this.policyEngine.getPolicyVersion(); + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java index c130309ea..aa0c80119 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java @@ -20,10 +20,14 @@ package org.apache.ranger.plugin.util; import java.io.File; +import java.io.FileFilter; import java.io.FileReader; import java.io.FileWriter; import java.io.Reader; import java.io.Writer; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; import java.util.Timer; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -449,14 +453,15 @@ public class PolicyRefresher extends Thread { } catch (Exception excp) { LOG.error("failed to save policies to cache file '" + cacheFile.getAbsolutePath() + "'", excp); } finally { - if(writer != null) { - try { - writer.close(); - } catch(Exception excp) { - LOG.error("error while closing opened cache file '" + cacheFile.getAbsolutePath() + "'", excp); - } - } - } + if (writer != null) { + try { + writer.close(); + deleteOldestVersionCacheFileInCacheDirectory(cacheFile.getParentFile()); + } catch (Exception excp) { + LOG.error("error while closing opened cache file '" + cacheFile.getAbsolutePath() + "'", excp); + } + } + } RangerPerfTracer.log(perf); @@ -490,6 +495,51 @@ public class PolicyRefresher extends Thread { } } + private void deleteOldestVersionCacheFileInCacheDirectory(File cacheDirectory) { + int maxVersionsToPreserve = plugIn.getConfig().getInt(plugIn.getConfig().getPropertyPrefix() + "max.versions.to.preserve", 1); + FileFilter logFileFilter = (file) -> file.getName().matches(".+json_.+"); + + File[] filesInParent = cacheDirectory.listFiles(logFileFilter); + List<Long> policyVersions = new ArrayList<>(); + + if (filesInParent != null && filesInParent.length > 0) { + for (File f : filesInParent) { + String fileName = f.getName(); + // Extract the part after json_ + int policyVersionIdx = fileName.lastIndexOf("json_"); + String policyVersionStr = fileName.substring(policyVersionIdx + 5); + Long policyVersion = Long.valueOf(policyVersionStr); + policyVersions.add(policyVersion); + } + } else { + LOG.info("No files matching '.+json_*' found"); + } + + if (!policyVersions.isEmpty()) { + policyVersions.sort(new Comparator<Long>() { + @Override + public int compare(Long o1, Long o2) { + if (o1.equals(o2)) return 0; + return o1 < o2 ? -1 : 1; + } + }); + } + + if (policyVersions.size() > maxVersionsToPreserve) { + String fileName = this.cacheFileName + "_" + Long.toString(policyVersions.get(0)); + String pathName = cacheDirectory.getAbsolutePath() + File.separator + fileName; + File toDelete = new File(pathName); + if (toDelete.exists()) { + boolean isDeleted = toDelete.delete(); + if (LOG.isDebugEnabled()) { + LOG.debug("file :[" + pathName + "] is deleted"); + } + } else { + LOG.info("File: " + pathName + " does not exist!"); + } + } + } + private void disableCache() { if (LOG.isDebugEnabled()) { LOG.debug("==> PolicyRefresher.disableCache(serviceName=" + serviceName + ")"); diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java index 9fda659ac..c768f1c34 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java @@ -33,9 +33,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileFilter; import java.io.FileWriter; import java.io.Writer; import java.util.ArrayList; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -203,6 +205,7 @@ public class RangerServicePoliciesCache { if (policies != null) { RangerAdminConfig config = RangerAdminConfig.getInstance(); boolean doSaveToDisk = config.getBoolean("ranger.admin.policy.save.to.disk", false); + int maxVersionsToSaveToDisk = config.getInt("ranger.admin.policy.max.versions.to.save.to.disk", 1); if (doSaveToDisk) { File cacheFile = null; @@ -237,6 +240,47 @@ public class RangerServicePoliciesCache { } catch (Exception excp) { LOG.error("failed to save policies to cache file '" + cacheFile.getAbsolutePath() + "'", excp); } + String serviceDefName = policies.getServiceDef().getName(); + String serviceName = policies.getServiceName(); + + File parentFile = cacheFile.getParentFile(); + FileFilter logFileFilter = (file) -> file.getName().matches(serviceDefName +"_.+json_.+"); + File[] filesInParent = parentFile.listFiles(logFileFilter); + List<Long> policyVersions = new ArrayList<>(); + if (filesInParent != null && filesInParent.length > 0) { + for (File f : filesInParent) { + String fileName = f.getName(); + // Extract the part after json_ + int policyVersionIdx = fileName.lastIndexOf("json_"); + String policyVersionStr = fileName.substring(policyVersionIdx + 5); + Long policyVersion = Long.valueOf(policyVersionStr); + policyVersions.add(policyVersion); + } + } else { + LOG.info("No files matching '" + serviceDefName + "_.+json_*' found"); + } + if (!policyVersions.isEmpty()) { + policyVersions.sort(new Comparator<Long>() { + @Override + public int compare(Long o1, Long o2) { + if (o1.equals(o2)) return 0; + return o1 < o2 ? -1 : 1; + } + }); + } + + if (policyVersions.size() > maxVersionsToSaveToDisk) { + String fileName = serviceDefName + "_" + serviceName + ".json_" + Long.toString(policyVersions.get(0)); + String pathName = parentFile.getAbsolutePath() + File.separator + fileName; + File toDelete = new File(pathName); + if (toDelete.exists()) { + //LOG.info("Deleting file :[" + pathName + "]"); + boolean isDeleted = toDelete.delete(); + //LOG.info("file :[" + pathName + "] is deleted"); + } else { + LOG.info("File: " + pathName + " does not exist!"); + } + } } } } else {