This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new eccbcc2a012 fix mods file metrics (#15284)
eccbcc2a012 is described below
commit eccbcc2a0126dfdffe32863a04864e0cd7625b22
Author: shuwenwei <[email protected]>
AuthorDate: Mon Apr 7 18:55:22 2025 +0800
fix mods file metrics (#15284)
---
.../iotdb/db/service/metrics/FileMetrics.java | 10 ++++
.../db/service/metrics/file/ModsFileMetrics.java | 4 +-
.../dataregion/tsfile/TsFileResource.java | 1 -
.../compaction/ReadChunkInnerCompactionTest.java | 70 ++++++++++++++++++++++
4 files changed, 82 insertions(+), 3 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
index 7f94b6cb364..8eca2ba3935 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
@@ -92,6 +92,16 @@ public class FileMetrics implements IMetricSet {
MODS_FILE_METRICS.decreaseModFileSize(size);
}
+ @TestOnly
+ public int getModFileNum() {
+ return MODS_FILE_METRICS.getModFileNum();
+ }
+
+ @TestOnly
+ public long getModFileSize() {
+ return MODS_FILE_METRICS.getModFileSize();
+ }
+
// endregion
public Map<Integer, Long> getRegionSizeMap() {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ModsFileMetrics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ModsFileMetrics.java
index 8ddefeddd24..ddc1a7d610e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ModsFileMetrics.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ModsFileMetrics.java
@@ -60,11 +60,11 @@ public class ModsFileMetrics implements IMetricSet {
MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(),
Tag.NAME.toString(), MODS);
}
- private int getModFileNum() {
+ public int getModFileNum() {
return modFileNum.get();
}
- private long getModFileSize() {
+ public long getModFileSize() {
return modFileSize.get();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index a90aaf72474..2d2fbea1e74 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -734,7 +734,6 @@ public class TsFileResource implements PersistentResource {
public void closeWithoutSettingStatus() throws IOException {
if (exclusiveModFile != null) {
exclusiveModFile.close();
- exclusiveModFile = null;
}
if (compactionModFile != null) {
compactionModFile.close();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
index 11021c51566..9d02499f28f 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
@@ -25,11 +25,13 @@ import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.service.metrics.FileMetrics;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import
org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
import
org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -38,6 +40,7 @@ import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.enums.CompressionType;
@@ -65,6 +68,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
import static
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter;
import static
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createCompressionType;
@@ -843,6 +847,72 @@ public class ReadChunkInnerCompactionTest extends
AbstractCompactionTest {
validateTargetDatas(sourceDatas, tsDataTypes);
}
+ @Test
+ public void testCascadedDeletionDuringCompaction() throws IOException,
InterruptedException {
+ TsFileResource source = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(source)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1"),
+ new TimeRange[] {new TimeRange(10, 20)},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ CountDownLatch latch1 = new CountDownLatch(1);
+ CountDownLatch latch2 = new CountDownLatch(1);
+ InnerSpaceCompactionTask task =
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ Collections.singletonList(source),
+ true,
+ new TestReadChunkCompactionPerformer(latch1, latch2),
+ 0);
+ new Thread(
+ () -> {
+ try {
+ latch1.await();
+ try (ModificationFile modificationFile =
source.getModFileForWrite()) {
+ modificationFile.write(
+ new TreeDeletionEntry(new
MeasurementPath("root.testsg.d1.s1"), 15));
+ }
+ latch2.countDown();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .start();
+ Assert.assertTrue(task.start());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(true).size());
+ tsFileManager.getTsFileList(true).get(0).getExclusiveModFile();
+ Assert.assertEquals(1, FileMetrics.getInstance().getModFileNum());
+ }
+
+ private static class TestReadChunkCompactionPerformer extends
ReadChunkCompactionPerformer {
+
+ private final CountDownLatch latch1;
+ private final CountDownLatch latch2;
+
+ public TestReadChunkCompactionPerformer(CountDownLatch latch1,
CountDownLatch latch2) {
+ this.latch1 = latch1;
+ this.latch2 = latch2;
+ }
+
+ @Override
+ public void perform()
+ throws IOException,
+ MetadataException,
+ InterruptedException,
+ StorageEngineException,
+ PageException {
+ super.perform();
+ latch1.countDown();
+ latch2.await();
+ }
+ }
+
private void writeEmptyAlignedChunk(
AlignedChunkWriterImpl alignedChunkWriter,
TsFileIOWriter tsFileIOWriter,
