This is an automated email from the ASF dual-hosted git repository. abhay pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push: new 04c93b3df RANGER-4565: Enhance Ranger's performance tracing module to optionally collect statistical information 04c93b3df is described below commit 04c93b3df9577c7f6e4f91f573a87c046311e15c Author: Abhay Kulkarni <ab...@apache.org> AuthorDate: Tue Nov 28 09:47:03 2023 -0800 RANGER-4565: Enhance Ranger's performance tracing module to optionally collect statistical information --- .../ranger/plugin/service/RangerBasePlugin.java | 6 ++ .../ranger/plugin/util/PerfDataRecorder.java | 77 +++++++++++++++++----- .../plugin/util/RangerPerfCollectorTracer.java | 2 +- .../ranger/plugin/util/RangerPerfTracer.java | 4 +- 4 files changed, 71 insertions(+), 18 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 2f4af9763..5d6c3d97c 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -86,6 +86,12 @@ public class RangerBasePlugin { this.pluginConfig = pluginConfig; this.pluginContext = new RangerPluginContext(pluginConfig); + boolean usePerfDataRecorder = pluginConfig.getBoolean("ranger.perf.aggregate.data", false); + int perfDataDumpInterval = pluginConfig.getInt("ranger.perf.aggregate.data.dump.interval", 0); + boolean usePerfDataLock = pluginConfig.getBoolean("ranger.perf.aggregate.data.lock.enabled", false); + + PerfDataRecorder.initialize(usePerfDataRecorder, perfDataDumpInterval, usePerfDataLock, null); + Set<String> superUsers = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".super.users")); Set<String> superGroups = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".super.groups")); Set<String> auditExcludeUsers = toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".audit.exclude.users")); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java index dce60b0ba..a1df53fac 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java @@ -37,13 +37,27 @@ public class PerfDataRecorder { private static final Logger PERF = RangerPerfTracer.getPerfLogger(PerfDataRecorder.class); private static volatile PerfDataRecorder instance; - private Map<String, PerfStatistic> perfStatistics = new HashMap<>(); + final private Map<String, PerfStatistic> perfStatistics = Collections.synchronizedMap(new HashMap<>()); + private RangerReadWriteLock lock = null; public static void initialize(List<String> names) { - if (instance == null) { - synchronized (PerfDataRecorder.class) { - if (instance == null) { - instance = new PerfDataRecorder(names); + initialize(true, 0, false, names); + } + + public static void initialize(final boolean useRecorder, final int collectionIntervalInSeconds, final boolean usePerfDataLock, List<String> names) { + if (useRecorder) { + if (instance == null) { + synchronized (PerfDataRecorder.class) { + if (instance == null) { + instance = new PerfDataRecorder(names); + instance.lock = new RangerReadWriteLock(usePerfDataLock); + if (collectionIntervalInSeconds > 0) { + Thread statDumper = new StatisticsDumper(collectionIntervalInSeconds); + statDumper.setName("Perf-Statistics-Dumper"); + statDumper.setDaemon(true); + statDumper.start(); + } + } } } } @@ -61,7 +75,9 @@ public class PerfDataRecorder { public static void clearStatistics() { if (instance != null) { - instance.clear(); + try (RangerReadWriteLock.RangerLock writeLock = instance.lock.getWriteLock()) { + instance.clear(); + } } } @@ -72,7 +88,11 @@ public class PerfDataRecorder { } private void dumpStatistics() { - List<String> tags = new ArrayList<>(perfStatistics.keySet()); + List<String> tags; + + try (RangerReadWriteLock.RangerLock readLock = lock.getReadLock()) { + tags = new ArrayList<>(perfStatistics.keySet()); + } Collections.sort(tags); @@ -111,20 +131,24 @@ public class PerfDataRecorder { } private void record(String tag, long cpuTime, long elapsedTime) { - PerfStatistic perfStatistic = perfStatistics.get(tag); + try (RangerReadWriteLock.RangerLock writeLock = lock.getWriteLock()) { + + PerfStatistic perfStatistic = perfStatistics.get(tag); - if (perfStatistic == null) { - synchronized (PerfDataRecorder.class) { - perfStatistic = perfStatistics.get(tag); + if (perfStatistic == null) { + synchronized (PerfDataRecorder.class) { + perfStatistic = perfStatistics.get(tag); - if(perfStatistic == null) { - perfStatistic = new PerfStatistic(); - perfStatistics.put(tag, perfStatistic); + if (perfStatistic == null) { + perfStatistic = new PerfStatistic(); + perfStatistics.put(tag, perfStatistic); + } } } - } - perfStatistic.addPerfDataItem(cpuTime, elapsedTime); + perfStatistic.addPerfDataItem(cpuTime, elapsedTime); + + } } private PerfDataRecorder(List<String> names) { @@ -206,4 +230,25 @@ public class PerfDataRecorder { return maxTimeSpent.get(); } } + + private static class StatisticsDumper extends Thread { + final int collectionIntervalInSeconds; + StatisticsDumper(final int collectionIntervalInSeconds) { + this.collectionIntervalInSeconds = collectionIntervalInSeconds; + } + @Override + public void run() { + while (true) { + try { + sleep(collectionIntervalInSeconds * 1000); + printStatistics(); + clearStatistics(); + } catch (InterruptedException exception) { + printStatistics(); + LOG.warn("Thread[" + this.getName() + "] was interrupted. Returning from thread. Performance statistics will NOT be dumped periodically!!"); + break; + } + } + } + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java index ebfebdae8..bc27ab24a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java @@ -37,7 +37,7 @@ public class RangerPerfCollectorTracer extends RangerPerfTracer { long reportingThreshold = threadInfo == null ? 0L : (1000000/1000 - 1); // just about a microsecond if (elapsedTime > reportingThreshold) { - logger.debug("[PERF]:" + (threadInfo != null ? threadInfo.getThreadName() : "") + ":" + tag + data + ":" + getElapsedCpuTime() + ":" + getElapsedUserTime()); + PerfDataRecorder.recordStatistic(tag, (getElapsedCpuTime()+500)/1000, (getElapsedUserTime() + 500)/1000); } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java index c8804739f..040b340e3 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java @@ -55,7 +55,9 @@ public class RangerPerfTracer { int indexOfTagEndMarker = StringUtils.indexOf(tag, tagEndMarker); if (indexOfTagEndMarker != -1) { realTag = StringUtils.substring(tag, 0, indexOfTagEndMarker); - data = StringUtils.substring(tag, indexOfTagEndMarker); + if (!PerfDataRecorder.collectStatistics()) { + data = StringUtils.substring(tag, indexOfTagEndMarker); + } } else { realTag = tag; }