This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 523f4788a17 Remove unnecessary metrics and optimize allocation byte
counting (#10738)
523f4788a17 is described below
commit 523f4788a171fe74491be9e8ccefdfb56fe61ca4
Author: Peng Junzhi <[email protected]>
AuthorDate: Sun Jul 30 21:57:54 2023 +0800
Remove unnecessary metrics and optimize allocation byte counting (#10738)
---
.../iotdb/metrics/metricsets/jvm/JvmGcMetrics.java | 148 ++++++++++-----------
.../apache/iotdb/metrics/utils/SystemMetric.java | 8 +-
2 files changed, 72 insertions(+), 84 deletions(-)
diff --git
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmGcMetrics.java
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmGcMetrics.java
index 413ffba13a8..3aeaa4e65bb 100644
---
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmGcMetrics.java
+++
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmGcMetrics.java
@@ -54,7 +54,7 @@ import java.util.concurrent.atomic.AtomicLong;
public class JvmGcMetrics implements IMetricSet, AutoCloseable {
private static final Logger logger =
LoggerFactory.getLogger(JvmGcMetrics.class);
private final List<Runnable> notificationListenerCleanUpRunnables = new
CopyOnWriteArrayList<>();
- private String youngGenPoolName;
+ private String firstYoungGenPoolName;
private String oldGenPoolName;
private String nonGenerationalMemoryPool;
private final Map<String, AtomicLong> lastGcTotalDurationMap = new
ConcurrentHashMap<>();
@@ -62,8 +62,8 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
public JvmGcMetrics() {
for (MemoryPoolMXBean mbean : ManagementFactory.getMemoryPoolMXBeans()) {
String name = mbean.getName();
- if (isYoungGenPool(name)) {
- youngGenPoolName = name;
+ if (isFirstYoungGenPool(name)) {
+ firstYoungGenPoolName = name;
} else if (isOldGenPool(name)) {
oldGenPoolName = name;
} else if (isNonGenerationalHeapPool(name)) {
@@ -98,12 +98,8 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
}
}
- private static boolean isConcurrentPhase(String cause, String name) {
- return "No GC".equals(cause) || "Shenandoah Cycles".equals(name);
- }
-
- private static boolean isYoungGenPool(String name) {
- return name != null && (name.endsWith("Eden Space") ||
name.endsWith("Survivor Space"));
+ private static boolean isFirstYoungGenPool(String name) {
+ return name != null && name.endsWith("Eden Space");
}
private static boolean isOldGenPool(String name) {
@@ -142,26 +138,35 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
liveDataSize,
AtomicLong::get);
- AtomicLong heapMemUsedPercentage = new
AtomicLong(calculateMemoryUsagePercentage());
- metricService.createAutoGauge(
- SystemMetric.JVM_GC_MEMORY_USED_PERCENT.toString(),
- MetricLevel.CORE,
- heapMemUsedPercentage,
- AtomicLong::get);
-
- Counter allocatedBytes =
- metricService.getOrCreateCounter(
- SystemMetric.JVM_GC_MEMORY_ALLOCATED_BYTES.toString(),
MetricLevel.CORE);
-
Counter promotedBytes =
(oldGenPoolName == null)
? null
: metricService.getOrCreateCounter(
SystemMetric.JVM_GC_MEMORY_PROMOTED_BYTES.toString(),
MetricLevel.CORE);
- // start watching for GC notifications
- final AtomicLong heapPoolSizeAfterGc = new AtomicLong();
+ Counter nonGenAllocatedBytes =
+ (nonGenerationalMemoryPool == null)
+ ? null
+ : metricService.getOrCreateCounter(
+ SystemMetric.JVM_GC_NON_GEN_MEMORY_ALLOCATED_BYTES.toString(),
MetricLevel.CORE);
+ Counter oldGenAllocatedBytes =
+ (oldGenPoolName == null)
+ ? null
+ : metricService.getOrCreateCounter(
+ SystemMetric.JVM_GC_OLD_MEMORY_ALLOCATED_BYTES.toString(),
MetricLevel.CORE);
+
+ Counter youngGenAllocatedBytes =
+ (firstYoungGenPoolName == null)
+ ? null
+ : metricService.getOrCreateCounter(
+ SystemMetric.JVM_GC_YOUNG_MEMORY_ALLOCATED_BYTES.toString(),
MetricLevel.CORE);
+
+ final AtomicLong firstYoungHeapPoolSizeAfterGc = new AtomicLong();
+ // long live heap pool includes old gen heap pool and non-generation heap
pool.
+ final AtomicLong longLivedHeapPoolSizeAfterGc = new AtomicLong();
+
+ // start watching for GC notifications
for (GarbageCollectorMXBean mbean :
ManagementFactory.getGarbageCollectorMXBeans()) {
if (!(mbean instanceof NotificationEmitter)) {
continue;
@@ -194,16 +199,15 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
previousTotal.set(total);
}
- String timerName;
- if (isConcurrentPhase(gcCause, notificationInfo.getGcName())) {
- timerName = "jvm_gc_concurrent_phase_time";
- } else {
- timerName = "jvm_gc_pause";
- }
// create a timer with tags named by gcCause, which binds gcCause
with gcDuration
Timer timer =
metricService.getOrCreateTimer(
- timerName, MetricLevel.CORE, "action", gcAction, "cause",
gcCause);
+ SystemMetric.JVM_GC_PAUSE.toString(),
+ MetricLevel.CORE,
+ "action",
+ gcAction,
+ "cause",
+ gcCause);
timer.update(duration, TimeUnit.MILLISECONDS);
// add support for ZGC
@@ -219,25 +223,6 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
pausesCount.inc();
}
- // monitoring old/young GC count, which is helpful for users to
locate GC exception.
- // Unfortunately, the JMX doesn't seem to provide an api for
monitoring mixed gc in G1.
- // In fact, JMX may treat mixed GCs as minor GCs.
- if (GcGenerationAge.fromName(notificationInfo.getGcName()) ==
GcGenerationAge.OLD) {
- Counter oldGcCounter =
- metricService.getOrCreateCounter(
- SystemMetric.JVM_GC_YOUNG_GC_COUNT.toString(),
MetricLevel.CORE);
- oldGcCounter.inc();
- } else if (GcGenerationAge.fromName(notificationInfo.getGcName())
- == GcGenerationAge.YOUNG) {
- Counter youngGcCounter =
- metricService.getOrCreateCounter(
- SystemMetric.JVM_GC_OLD_GC_COUNT.toString(),
MetricLevel.CORE);
- youngGcCounter.inc();
- }
-
- // update memory usage percentage
- heapMemUsedPercentage.set(calculateMemoryUsagePercentage());
-
// Update promotion and allocation counters
final Map<String, MemoryUsage> before =
gcInfo.getMemoryUsageBeforeGc();
final Map<String, MemoryUsage> after =
gcInfo.getMemoryUsageAfterGc();
@@ -246,8 +231,8 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
countPoolSizeDelta(
gcInfo.getMemoryUsageBeforeGc(),
gcInfo.getMemoryUsageAfterGc(),
- allocatedBytes,
- heapPoolSizeAfterGc,
+ nonGenAllocatedBytes,
+ longLivedHeapPoolSizeAfterGc,
nonGenerationalMemoryPool);
if (after.get(nonGenerationalMemoryPool).getUsed()
< before.get(nonGenerationalMemoryPool).getUsed()) {
@@ -272,8 +257,7 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
// GC (since in JMX, a minor GC of G1 may actually represent
mixed GC, which collect
// some obj in old gen region). To track the
// live data size we record the value if we see a reduction in
the old gen heap size
- // or
- // after a major GC.
+ // or after a major GC.
if (oldAfter < oldBefore
|| GcGenerationAge.fromName(notificationInfo.getGcName())
== GcGenerationAge.OLD) {
@@ -281,15 +265,21 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
final long oldMaxAfter = after.get(oldGenPoolName).getMax();
maxDataSize.set(oldMaxAfter);
}
+ countPoolSizeDelta(
+ gcInfo.getMemoryUsageBeforeGc(),
+ gcInfo.getMemoryUsageAfterGc(),
+ oldGenAllocatedBytes,
+ longLivedHeapPoolSizeAfterGc,
+ oldGenPoolName);
}
- if (youngGenPoolName != null) {
+ if (firstYoungGenPoolName != null) {
countPoolSizeDelta(
gcInfo.getMemoryUsageBeforeGc(),
gcInfo.getMemoryUsageAfterGc(),
- allocatedBytes,
- heapPoolSizeAfterGc,
- youngGenPoolName);
+ youngGenAllocatedBytes,
+ firstYoungHeapPoolSizeAfterGc,
+ firstYoungGenPoolName);
}
}
};
@@ -321,12 +311,22 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
metricService.remove(MetricType.AUTO_GAUGE,
SystemMetric.JVM_GC_MAX_DATA_SIZE_BYTES.toString());
metricService.remove(
MetricType.AUTO_GAUGE,
SystemMetric.JVM_GC_LIVE_DATA_SIZE_BYTES.toString());
- metricService.remove(MetricType.COUNTER,
SystemMetric.JVM_GC_MEMORY_ALLOCATED_BYTES.toString());
- metricService.remove(MetricType.AUTO_GAUGE,
SystemMetric.JVM_GC_MEMORY_USED_PERCENT.toString());
- if (oldGenPoolName != null) {
+ if (nonGenerationalMemoryPool != null) {
metricService.remove(
- MetricType.COUNTER,
SystemMetric.JVM_GC_MEMORY_PROMOTED_BYTES.toString());
+ MetricType.COUNTER,
SystemMetric.JVM_GC_NON_GEN_MEMORY_ALLOCATED_BYTES.toString());
+ } else {
+ if (oldGenPoolName != null) {
+ metricService.remove(
+ MetricType.COUNTER,
SystemMetric.JVM_GC_MEMORY_PROMOTED_BYTES.toString());
+ metricService.remove(
+ MetricType.COUNTER,
SystemMetric.JVM_GC_OLD_MEMORY_ALLOCATED_BYTES.toString());
+ }
+
+ if (firstYoungGenPoolName != null) {
+ metricService.remove(
+ MetricType.COUNTER,
SystemMetric.JVM_GC_YOUNG_MEMORY_ALLOCATED_BYTES.toString());
+ }
}
// start watching for GC notifications
@@ -342,13 +342,14 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
String gcCause = notificationInfo.getGcCause();
String gcAction = notificationInfo.getGcAction();
- String timerName;
- if (isConcurrentPhase(gcCause, notificationInfo.getGcName())) {
- timerName = "jvm_gc_concurrent_phase_time";
- } else {
- timerName = "jvm_gc_pause";
- }
- metricService.remove(MetricType.TIMER, timerName, "action",
gcAction, "cause", gcCause);
+ metricService.remove(
+ MetricType.TIMER,
+ SystemMetric.JVM_GC_PAUSE.toString(),
+ "action",
+ gcAction,
+ "cause",
+ gcCause);
+
if (mbean.getName().equals("ZGC Cycles")) {
metricService.remove(
MetricType.COUNTER,
SystemMetric.JVM_ZGC_CYCLES_COUNT.toString());
@@ -356,13 +357,6 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
metricService.remove(
MetricType.COUNTER,
SystemMetric.JVM_ZGC_PAUSES_COUNT.toString());
}
- if (GcGenerationAge.fromName(notificationInfo.getGcName()) ==
GcGenerationAge.OLD) {
- metricService.remove(
- MetricType.COUNTER,
SystemMetric.JVM_GC_YOUNG_GC_COUNT.toString());
- } else if (GcGenerationAge.fromName(notificationInfo.getGcName())
- == GcGenerationAge.YOUNG) {
- metricService.remove(MetricType.COUNTER,
SystemMetric.JVM_GC_OLD_GC_COUNT.toString());
- }
};
NotificationEmitter notificationEmitter = (NotificationEmitter) mbean;
notificationEmitter.addNotificationListener(
@@ -426,12 +420,6 @@ public class JvmGcMetrics implements IMetricSet,
AutoCloseable {
notificationListenerCleanUpRunnables.forEach(Runnable::run);
}
- public long calculateMemoryUsagePercentage() {
- return (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed()
- * 100
- / Runtime.getRuntime().maxMemory());
- }
-
enum GcGenerationAge {
OLD,
YOUNG,
diff --git
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java
index 645c2307eea..e04586c0f11 100644
---
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java
+++
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java
@@ -79,13 +79,13 @@ public enum SystemMetric {
JVM_THREADS_STATUS_THREADS("jvm_threads_states_threads"),
JVM_GC_MAX_DATA_SIZE_BYTES("jvm_gc_max_data_size_bytes"),
JVM_GC_LIVE_DATA_SIZE_BYTES("jvm_gc_live_data_size_bytes"),
- JVM_GC_MEMORY_ALLOCATED_BYTES("jvm_gc_memory_allocated_bytes"),
+ JVM_GC_YOUNG_MEMORY_ALLOCATED_BYTES("jvm_gc_young_memory_allocated_bytes"),
+ JVM_GC_OLD_MEMORY_ALLOCATED_BYTES("jvm_gc_old_memory_allocated_bytes"),
+
JVM_GC_NON_GEN_MEMORY_ALLOCATED_BYTES("jvm_gc_non_gen_memory_allocated_bytes"),
JVM_GC_MEMORY_PROMOTED_BYTES("jvm_gc_memory_promoted_bytes"),
- JVM_GC_MEMORY_USED_PERCENT("jvm_gc_memory_used_percent"),
+ JVM_GC_PAUSE("jvm_gc_pause"),
JVM_ZGC_CYCLES_COUNT("jvm_zgc_cycles_count"),
JVM_ZGC_PAUSES_COUNT("jvm_zgc_pauses_count"),
- JVM_GC_YOUNG_GC_COUNT("jvm_gc_old_gc_count"),
- JVM_GC_OLD_GC_COUNT("jvm_gc_young_gc_count"),
JVM_GC_ACCUMULATED_TIME_PERCENTAGE("jvm_gc_accumulated_time_percentage"),
// net related
RECEIVED_BYTES("received_bytes"),
