This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch add-level-file-metric
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/add-level-file-metric by this
push:
new c3f4bed686e add level file metrics
c3f4bed686e is described below
commit c3f4bed686edf15559baf23bec4063712ec017b5
Author: Liu Xuxin <[email protected]>
AuthorDate: Thu May 25 19:12:35 2023 +0800
add level file metrics
---
.../iotdb/db/engine/TsFileMetricManager.java | 74 +++++++++++++++++++++-
.../execute/task/CrossSpaceCompactionTask.java | 12 +++-
.../execute/task/InnerSpaceCompactionTask.java | 12 +++-
.../iotdb/db/engine/storagegroup/DataRegion.java | 41 +++++++++---
.../iotdb/db/service/metrics/FileMetrics.java | 1 +
5 files changed, 125 insertions(+), 15 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/TsFileMetricManager.java
b/server/src/main/java/org/apache/iotdb/db/engine/TsFileMetricManager.java
index a3cf6c83361..1b7a1e600ad 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/TsFileMetricManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/TsFileMetricManager.java
@@ -19,19 +19,34 @@
package org.apache.iotdb.db.engine;
+import org.apache.iotdb.commons.service.metric.enums.Tag;
import
org.apache.iotdb.db.engine.compaction.execute.task.AbstractCompactionTask;
import
org.apache.iotdb.db.engine.compaction.execute.task.CompactionTaskSummary;
import
org.apache.iotdb.db.engine.compaction.execute.task.InnerSpaceCompactionTask;
import org.apache.iotdb.db.engine.compaction.schedule.CompactionTaskManager;
+import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.service.metrics.FileMetrics;
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
/** This class collect the number and size of tsfile, and send it to the
{@link FileMetrics} */
public class TsFileMetricManager {
+ private static final Logger log =
LoggerFactory.getLogger(TsFileMetricManager.class);
private static final TsFileMetricManager INSTANCE = new
TsFileMetricManager();
+ private static final String FILE_LEVEL_COUNT = "file_level_count";
+ private static final String SEQUENCE = "sequence";
+ private static final String UNSEQUENCE = "unsequence";
+ private static final String LEVEL = "level";
private final AtomicLong seqFileSize = new AtomicLong(0);
private final AtomicLong unseqFileSize = new AtomicLong(0);
private final AtomicInteger seqFileNum = new AtomicInteger(0);
@@ -40,6 +55,8 @@ public class TsFileMetricManager {
private final AtomicInteger modFileNum = new AtomicInteger(0);
private final AtomicLong modFileSize = new AtomicLong(0);
+ private final Map<Integer, Integer> seqLevelTsFileCountMap = new HashMap<>();
+ private final Map<Integer, Integer> unseqLevelTsFileCountMap = new
HashMap<>();
private long lastUpdateTime = 0;
private static final long UPDATE_INTERVAL = 10_000L;
@@ -50,6 +67,7 @@ public class TsFileMetricManager {
private final AtomicInteger innerSeqCompactionTempFileNum = new
AtomicInteger(0);
private final AtomicInteger innerUnseqCompactionTempFileNum = new
AtomicInteger(0);
private final AtomicInteger crossCompactionTempFileNum = new
AtomicInteger(0);
+ private AbstractMetricService metricService;
private TsFileMetricManager() {}
@@ -57,7 +75,11 @@ public class TsFileMetricManager {
return INSTANCE;
}
- public void addFile(long size, boolean seq) {
+ public void setMetricService(AbstractMetricService metricService) {
+ this.metricService = metricService;
+ }
+
+ public void addFile(long size, boolean seq, String name) {
if (seq) {
seqFileSize.getAndAdd(size);
seqFileNum.incrementAndGet();
@@ -65,9 +87,32 @@ public class TsFileMetricManager {
unseqFileSize.getAndAdd(size);
unseqFileNum.incrementAndGet();
}
+ try {
+ TsFileNameGenerator.TsFileName tsFileName =
TsFileNameGenerator.getTsFileName(name);
+ int level = tsFileName.getInnerCompactionCnt();
+ int count = -1;
+ if (seq) {
+ count = seqLevelTsFileCountMap.compute(level, (k, v) -> v == null ? 1
: v + 1);
+ } else {
+ count = unseqLevelTsFileCountMap.compute(level, (k, v) -> v == null ?
1 : v + 1);
+ }
+ if (metricService != null) {
+ metricService
+ .getOrCreateGauge(
+ FILE_LEVEL_COUNT,
+ MetricLevel.CORE,
+ Tag.TYPE.toString(),
+ seq ? SEQUENCE : UNSEQUENCE,
+ LEVEL,
+ String.valueOf(level))
+ .set(count);
+ }
+ } catch (IOException e) {
+ log.error("Unexpected error occurred when getting tsfile name", e);
+ }
}
- public void deleteFile(long size, boolean seq, int num) {
+ public void deleteFile(long size, boolean seq, int num, List<String> names) {
if (seq) {
seqFileSize.getAndAdd(-size);
seqFileNum.getAndAdd(-num);
@@ -75,6 +120,31 @@ public class TsFileMetricManager {
unseqFileSize.getAndAdd(-size);
unseqFileNum.getAndAdd(-num);
}
+ for (String name : names) {
+ int level = -1;
+ int count = -1;
+ try {
+ TsFileNameGenerator.TsFileName tsFileName =
TsFileNameGenerator.getTsFileName(name);
+ level = tsFileName.getInnerCompactionCnt();
+ count =
+ seq
+ ? seqLevelTsFileCountMap.compute(level, (k, v) -> v == null ?
0 : v - 1)
+ : unseqLevelTsFileCountMap.compute(level, (k, v) -> v == null
? 0 : v - 1);
+ } catch (IOException e) {
+ log.error("Unexpected error occurred when getting tsfile name", e);
+ }
+ if (metricService != null && level != -1 && count != -1) {
+ metricService
+ .getOrCreateGauge(
+ FILE_LEVEL_COUNT,
+ MetricLevel.CORE,
+ Tag.TYPE.toString(),
+ seq ? SEQUENCE : UNSEQUENCE,
+ LEVEL,
+ String.valueOf(level))
+ .set(count);
+ }
+ }
}
public long getFileSize(boolean seq) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
index 4aa9d378d33..67111d8229b 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -212,7 +212,9 @@ public class CrossSpaceCompactionTask extends
AbstractCompactionTask {
// update the metrics finally in case of any exception occurs
for (TsFileResource targetResource : targetTsfileResourceList) {
if (!targetResource.isDeleted()) {
-
TsFileMetricManager.getInstance().addFile(targetResource.getTsFileSize(), true);
+ TsFileMetricManager.getInstance()
+ .addFile(
+ targetResource.getTsFileSize(), true,
targetResource.getTsFile().getName());
// set target resources to CLOSED, so that they can be selected to
compact
targetResource.setStatus(TsFileResourceStatus.NORMAL);
@@ -221,10 +223,14 @@ public class CrossSpaceCompactionTask extends
AbstractCompactionTask {
targetResource.remove();
}
}
+ List<String> fileNames = new ArrayList<>();
+ selectedSequenceFiles.forEach(x ->
fileNames.add(x.getTsFile().getName()));
TsFileMetricManager.getInstance()
- .deleteFile(sequenceFileSize, true, selectedSequenceFiles.size());
+ .deleteFile(sequenceFileSize, true, selectedSequenceFiles.size(),
fileNames);
+ fileNames.clear();
+ selectedUnsequenceFiles.forEach(x ->
fileNames.add(x.getTsFile().getName()));
TsFileMetricManager.getInstance()
- .deleteFile(unsequenceFileSize, false,
selectedUnsequenceFiles.size());
+ .deleteFile(unsequenceFileSize, false,
selectedUnsequenceFiles.size(), fileNames);
CompactionMetrics.getInstance().recordSummaryInfo(summary);
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
index de28ae37c18..a9dcb0fbc78 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -236,7 +236,10 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
// inner space compaction task has only one target file
if (!targetTsFileResource.isDeleted()) {
TsFileMetricManager.getInstance()
- .addFile(targetTsFileResource.getTsFile().length(), sequence);
+ .addFile(
+ targetTsFileResource.getTsFile().length(),
+ sequence,
+ targetTsFileResource.getTsFile().getName());
// set target resource to CLOSED, so that it can be selected to
compact
targetTsFileResource.setStatus(TsFileResourceStatus.NORMAL);
@@ -244,8 +247,13 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
// target resource is empty after compaction, then delete it
targetTsFileResource.remove();
}
+ List<String> fileNames = new ArrayList<>();
+ for (TsFileResource resource : selectedTsFileResourceList) {
+ fileNames.add(resource.getTsFile().getName());
+ }
TsFileMetricManager.getInstance()
- .deleteFile(totalSizeOfDeletedFile, sequence,
selectedTsFileResourceList.size());
+ .deleteFile(
+ totalSizeOfDeletedFile, sequence,
selectedTsFileResourceList.size(), fileNames);
CompactionMetrics.getInstance().recordSummaryInfo(summary);
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index dd90e5f2511..4d2131ce24d 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -470,7 +470,8 @@ public class DataRegion implements IDataRegionForQuery {
// tsFiles without resource file are unsealed
for (TsFileResource resource : value) {
if (resource.resourceFileExists()) {
-
TsFileMetricManager.getInstance().addFile(resource.getTsFile().length(), true);
+ TsFileMetricManager.getInstance()
+ .addFile(resource.getTsFile().length(), true,
resource.getTsFile().getName());
if (resource.getModFile().exists()) {
TsFileMetricManager.getInstance().increaseModFileNum(1);
TsFileMetricManager.getInstance()
@@ -496,7 +497,8 @@ public class DataRegion implements IDataRegionForQuery {
// tsFiles without resource file are unsealed
for (TsFileResource resource : value) {
if (resource.resourceFileExists()) {
-
TsFileMetricManager.getInstance().addFile(resource.getTsFile().length(), false);
+ TsFileMetricManager.getInstance()
+ .addFile(resource.getTsFile().length(), false,
resource.getTsFile().getName());
}
if (resource.getModFile().exists()) {
TsFileMetricManager.getInstance().increaseModFileNum(1);
@@ -797,7 +799,10 @@ public class DataRegion implements IDataRegionForQuery {
updateLastFlushTime(tsFileResource, isSeq);
tsFileResourceManager.registerSealedTsFileResource(tsFileResource);
TsFileMetricManager.getInstance()
- .addFile(tsFileResource.getTsFile().length(),
recoverPerformer.isSequence());
+ .addFile(
+ tsFileResource.getTsFile().length(),
+ recoverPerformer.isSequence(),
+ tsFileResource.getTsFile().getName());
} else {
// the last file is not closed, continue writing to it
RestorableTsFileIOWriter writer = recoverPerformer.getWriter();
@@ -1609,7 +1614,12 @@ public class DataRegion implements IDataRegionForQuery {
if (resource.tryWriteLock()) {
try {
// try to delete physical data file
- TsFileMetricManager.getInstance().deleteFile(resource.getTsFileSize(),
isSeq, 1);
+ TsFileMetricManager.getInstance()
+ .deleteFile(
+ resource.getTsFileSize(),
+ isSeq,
+ 1,
+ Collections.singletonList(resource.getTsFile().getName()));
resource.remove();
tsFileManager.remove(resource, isSeq);
logger.info(
@@ -2185,7 +2195,10 @@ public class DataRegion implements IDataRegionForQuery {
closeStorageGroupCondition.notifyAll();
}
TsFileMetricManager.getInstance()
- .addFile(tsFileProcessor.getTsFileResource().getTsFileSize(),
tsFileProcessor.isSequence());
+ .addFile(
+ tsFileProcessor.getTsFileResource().getTsFileSize(),
+ tsFileProcessor.isSequence(),
+ tsFileProcessor.getTsFileResource().getTsFile().getName());
logger.info("signal closing database condition in {}", databaseName + "-"
+ dataRegionId);
}
@@ -2387,7 +2400,11 @@ public class DataRegion implements IDataRegionForQuery {
}
loadTsFileToUnSequence(
tsfileToBeInserted, newTsFileResource, newFilePartitionId,
deleteOriginFile);
-
TsFileMetricManager.getInstance().addFile(newTsFileResource.getTsFile().length(),
false);
+ TsFileMetricManager.getInstance()
+ .addFile(
+ newTsFileResource.getTsFile().length(),
+ false,
+ newTsFileResource.getTsFile().getName());
resetLastCacheWhenLoadingTsFile(); // update last cache
updateLastFlushTime(newTsFileResource); // update last flush time
@@ -2724,7 +2741,11 @@ public class DataRegion implements IDataRegionForQuery {
tsFileResourceToBeMoved = sequenceResource;
tsFileManager.remove(tsFileResourceToBeMoved, true);
TsFileMetricManager.getInstance()
- .deleteFile(tsFileResourceToBeMoved.getTsFileSize(), true, 1);
+ .deleteFile(
+ tsFileResourceToBeMoved.getTsFileSize(),
+ true,
+ 1,
+
Collections.singletonList(tsFileResourceToBeMoved.getTsFile().getName()));
break;
}
}
@@ -2736,7 +2757,11 @@ public class DataRegion implements IDataRegionForQuery {
tsFileResourceToBeMoved = unsequenceResource;
tsFileManager.remove(tsFileResourceToBeMoved, false);
TsFileMetricManager.getInstance()
- .deleteFile(tsFileResourceToBeMoved.getTsFileSize(), false, 1);
+ .deleteFile(
+ tsFileResourceToBeMoved.getTsFileSize(),
+ false,
+ 1,
+
Collections.singletonList(tsFileResourceToBeMoved.getTsFile().getName()));
break;
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
b/server/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
index ec9b7775658..bc1954142c5 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
@@ -65,6 +65,7 @@ public class FileMetrics implements IMetricSet {
}
private void bindTsFileMetrics(AbstractMetricService metricService) {
+ TS_FILE_METRIC_MANAGER.setMetricService(metricService);
metricService.createAutoGauge(
Metric.FILE_SIZE.toString(),
MetricLevel.CORE,
