This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c93040e38d [core] Minor refactor for cache metrics
c93040e38d is described below
commit c93040e38d7b5d6636cc80f3629b8c185e675f80
Author: Jingsong <[email protected]>
AuthorDate: Wed Dec 18 10:48:33 2024 +0800
[core] Minor refactor for cache metrics
---
.../org/apache/paimon/catalog/CachingCatalog.java | 33 ++++++++++-------
.../paimon/operation/metrics/ScanMetrics.java | 41 +++++++++-------------
.../java/org/apache/paimon/utils/ObjectsCache.java | 11 +++---
.../java/org/apache/paimon/utils/ObjectsFile.java | 14 ++++----
.../org/apache/paimon/utils/SegmentsCache.java | 4 +--
5 files changed, 52 insertions(+), 51 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
index 5c9c785493..34e53f32f2 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java
@@ -312,14 +312,32 @@ public class CachingCatalog extends DelegateCatalog {
}
}
+ // ================================== Cache Public API
+ // ================================================
+
+ /**
+ * Partition cache will affect the latency of table, so refresh method is
provided for compute
+ * engine.
+ */
+ public void refreshPartitions(Identifier identifier) throws
TableNotExistException {
+ if (partitionCache != null) {
+ List<PartitionEntry> result = wrapped.listPartitions(identifier);
+ partitionCache.put(identifier, result);
+ }
+ }
+
+ /**
+ * Cache sizes for compute engine. This method can let the outside know
the specific usage of
+ * cache.
+ */
public CacheSizes estimatedCacheSizes() {
long databaseCacheSize = databaseCache.estimatedSize();
long tableCacheSize = tableCache.estimatedSize();
long manifestCacheSize = 0L;
long manifestCacheBytes = 0L;
if (manifestCache != null) {
- manifestCacheSize = manifestCache.getSegmentCacheSize();
- manifestCacheBytes = manifestCache.getSegmentCacheBytes();
+ manifestCacheSize = manifestCache.estimatedSize();
+ manifestCacheBytes = manifestCache.totalCacheBytes();
}
long partitionCacheSize = 0L;
if (partitionCache != null) {
@@ -336,18 +354,9 @@ public class CachingCatalog extends DelegateCatalog {
partitionCacheSize);
}
- // ================================== refresh
================================================
- // following caches will affect the latency of table, so refresh method is
provided for engine
-
- public void refreshPartitions(Identifier identifier) throws
TableNotExistException {
- if (partitionCache != null) {
- List<PartitionEntry> result = wrapped.listPartitions(identifier);
- partitionCache.put(identifier, result);
- }
- }
-
/** Cache sizes of a caching catalog. */
public static class CacheSizes {
+
private final long databaseCacheSize;
private final long tableCacheSize;
private final long manifestCacheSize;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java
b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java
index fdc41baf28..be6b514e04 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java
@@ -28,24 +28,6 @@ public class ScanMetrics {
private static final int HISTOGRAM_WINDOW_SIZE = 100;
public static final String GROUP_NAME = "scan";
-
- private final MetricGroup metricGroup;
- private Histogram durationHistogram;
-
- private ScanStats latestScan;
- private CacheMetrics cacheMetrics;
-
- public ScanMetrics(MetricRegistry registry, String tableName) {
- this.metricGroup = registry.tableMetricGroup(GROUP_NAME, tableName);
- this.cacheMetrics = new CacheMetrics();
- registerGenericScanMetrics();
- }
-
- @VisibleForTesting
- public MetricGroup getMetricGroup() {
- return metricGroup;
- }
-
public static final String LAST_SCAN_DURATION = "lastScanDuration";
public static final String SCAN_DURATION = "scanDuration";
public static final String LAST_SCANNED_MANIFESTS = "lastScannedManifests";
@@ -54,10 +36,18 @@ public class ScanMetrics {
public static final String MANIFEST_HIT_CACHE = "manifestHitCache";
public static final String MANIFEST_MISSED_CACHE = "manifestMissedCache";
- private void registerGenericScanMetrics() {
+ private final MetricGroup metricGroup;
+ private final Histogram durationHistogram;
+ private final CacheMetrics cacheMetrics;
+
+ private ScanStats latestScan;
+
+ public ScanMetrics(MetricRegistry registry, String tableName) {
+ metricGroup = registry.tableMetricGroup(GROUP_NAME, tableName);
metricGroup.gauge(
LAST_SCAN_DURATION, () -> latestScan == null ? 0L :
latestScan.getDuration());
durationHistogram = metricGroup.histogram(SCAN_DURATION,
HISTOGRAM_WINDOW_SIZE);
+ cacheMetrics = new CacheMetrics();
metricGroup.gauge(
LAST_SCANNED_MANIFESTS,
() -> latestScan == null ? 0L :
latestScan.getScannedManifests());
@@ -67,12 +57,13 @@ public class ScanMetrics {
metricGroup.gauge(
LAST_SCAN_RESULTED_TABLE_FILES,
() -> latestScan == null ? 0L :
latestScan.getResultedTableFiles());
- metricGroup.gauge(
- MANIFEST_HIT_CACHE,
- () -> cacheMetrics == null ? 0L :
cacheMetrics.getHitObject().get());
- metricGroup.gauge(
- MANIFEST_MISSED_CACHE,
- () -> cacheMetrics == null ? 0L :
cacheMetrics.getMissedObject().get());
+ metricGroup.gauge(MANIFEST_HIT_CACHE, () ->
cacheMetrics.getHitObject().get());
+ metricGroup.gauge(MANIFEST_MISSED_CACHE, () ->
cacheMetrics.getMissedObject().get());
+ }
+
+ @VisibleForTesting
+ MetricGroup getMetricGroup() {
+ return metricGroup;
}
public void reportScan(ScanStats scanStats) {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java
b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java
index 6f14c78107..8fe13943a3 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java
@@ -48,7 +48,8 @@ public class ObjectsCache<K, V> {
private final ThreadLocal<InternalRowSerializer> formatSerializer;
private final FunctionWithIOException<K, Long> fileSizeFunction;
private final BiFunctionWithIOE<K, Long, CloseableIterator<InternalRow>>
reader;
- private CacheMetrics cacheMetrics;
+
+ @Nullable private CacheMetrics cacheMetrics;
public ObjectsCache(
SegmentsCache<K> cache,
@@ -64,6 +65,10 @@ public class ObjectsCache<K, V> {
this.reader = reader;
}
+ public void withCacheMetrics(@Nullable CacheMetrics cacheMetrics) {
+ this.cacheMetrics = cacheMetrics;
+ }
+
public List<V> read(
K key,
@Nullable Long fileSize,
@@ -138,8 +143,4 @@ public class ObjectsCache<K, V> {
throw new RuntimeException(e);
}
}
-
- public void withCacheMetrics(CacheMetrics cacheMetrics) {
- this.cacheMetrics = cacheMetrics;
- }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsFile.java
b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsFile.java
index fc523ce239..39cf9e4b98 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsFile.java
@@ -77,6 +77,13 @@ public class ObjectsFile<T> implements SimpleFileReader<T> {
this::createIterator);
}
+ public ObjectsFile<T> withCacheMetrics(@Nullable CacheMetrics
cacheMetrics) {
+ if (cache != null) {
+ cache.withCacheMetrics(cacheMetrics);
+ }
+ return this;
+ }
+
public FileIO fileIO() {
return fileIO;
}
@@ -208,11 +215,4 @@ public class ObjectsFile<T> implements SimpleFileReader<T>
{
throw new RuntimeException(e);
}
}
-
- public ObjectsFile<T> withCacheMetrics(CacheMetrics cacheMetrics) {
- if (cache != null) {
- cache.withCacheMetrics(cacheMetrics);
- }
- return this;
- }
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java
b/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java
index 0ef4b13a31..8cac9f03be 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/SegmentsCache.java
@@ -91,11 +91,11 @@ public class SegmentsCache<T> {
return new SegmentsCache<>(pageSize, maxMemorySize, maxElementSize);
}
- public long getSegmentCacheSize() {
+ public long estimatedSize() {
return cache.estimatedSize();
}
- public long getSegmentCacheBytes() {
+ public long totalCacheBytes() {
return cache.asMap().entrySet().stream()
.mapToLong(entry -> weigh(entry.getKey(), entry.getValue()))
.sum();
