This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new aa1e322fa81 Fix duplicate deletion emission in TsFileSplitter (#17538)
aa1e322fa81 is described below
commit aa1e322fa8184441eb821e7a18ac60db05c83912
Author: Zhenyu Luo <[email protected]>
AuthorDate: Tue Apr 28 17:47:41 2026 +0800
Fix duplicate deletion emission in TsFileSplitter (#17538)
---
.../storageengine/load/splitter/DeletionData.java | 4 +
.../load/splitter/TsFileSplitter.java | 1 -
.../BatchedCompactionWithTsFileSplitterTest.java | 86 ++++++++++++++++++++++
3 files changed, 90 insertions(+), 1 deletion(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java
index 186426650fe..dc4908ef2c3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java
@@ -59,6 +59,10 @@ public class DeletionData implements TsFileData {
deletion.serializeWithoutFileOffset(stream);
}
+ public Deletion getModEntry() {
+ return this.deletion;
+ }
+
public static DeletionData deserialize(InputStream stream)
throws IllegalPathException, IOException {
return new DeletionData(Deletion.deserializeWithoutFileOffset(new
DataInputStream(stream)));
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java
index af401f84aa7..4571bdb1531 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java
@@ -156,7 +156,6 @@ public class TsFileSplitter {
long chunkOffset = reader.position();
timeChunkIndexOfCurrentValueColumn = pageIndex2TimesList.size();
consumeAllAlignedChunkData(chunkOffset, pageIndex2ChunkData);
- handleModification(offset2Deletions, chunkOffset);
ChunkHeader header = reader.readChunkHeader(marker);
String measurementId = header.getMeasurementID();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java
index 37d38411ff6..1a407648a51 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.compaction;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.load.LoadFileException;
@@ -30,10 +31,15 @@ import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionCheckerUtils;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
import org.apache.iotdb.db.storageengine.load.splitter.AlignedChunkData;
+import org.apache.iotdb.db.storageengine.load.splitter.DeletionData;
import org.apache.iotdb.db.storageengine.load.splitter.TsFileData;
import org.apache.iotdb.db.storageengine.load.splitter.TsFileSplitter;
@@ -54,7 +60,9 @@ import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
+import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -233,6 +241,84 @@ public class BatchedCompactionWithTsFileSplitterTest
extends AbstractCompactionT
consumeChunkDataAndValidate(targetResource);
}
+ @Test
+ public void testDeletionDataShouldOnlyBeGeneratedOnceAtEnd()
+ throws IOException, MetadataException, LoadFileException,
IllegalPathException {
+ TsFileResource resource = createAlignedMultiDeviceFile();
+ try (ModificationFile modificationFile =
ModificationFile.getNormalMods(resource)) {
+ modificationFile.write(
+ new Deletion(new MeasurementPath("root.testsg.d0.s0"),
Long.MAX_VALUE, 100));
+ modificationFile.write(
+ new Deletion(new MeasurementPath("root.testsg.d0.s1"),
Long.MAX_VALUE, 200));
+ modificationFile.write(
+ new Deletion(new MeasurementPath("root.testsg.d1.s0"),
Long.MAX_VALUE, 300));
+ modificationFile.write(
+ new Deletion(new MeasurementPath("root.testsg.d1.s1"),
Long.MAX_VALUE, 400));
+ }
+
+ List<Modification> expectedMods =
+ new
ArrayList<>(ModificationFile.getNormalMods(resource).getModifications());
+ List<Modification> deletionMods = new ArrayList<>();
+ File actualModsFile = new File(resource.getTsFilePath() + ".mods");
+ try (ModificationFile actualModificationFile =
+ new ModificationFile(actualModsFile.getAbsolutePath())) {
+ TsFileSplitter splitter =
+ new TsFileSplitter(
+ resource.getTsFile(),
+ tsFileData -> {
+ if (tsFileData instanceof DeletionData) {
+ deletionMods.add(((DeletionData) tsFileData).getModEntry());
+ }
+ return true;
+ });
+ splitter.splitTsFileByDataPartition();
+ }
+
+ List<Modification> actualMods;
+ try (ModificationFile actualModificationFile =
+ new ModificationFile(actualModsFile.getAbsolutePath())) {
+ actualMods = new ArrayList<>(actualModificationFile.getModifications());
+ }
+ Assert.assertEquals(expectedMods, actualMods);
+ Files.deleteIfExists(actualModsFile.toPath());
+ }
+
+ private TsFileResource createAlignedMultiDeviceFile() throws IOException {
+ TsFileResource resource = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(resource)) {
+ writer.startChunkGroup("d0");
+ writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue(
+ Arrays.asList("s0", "s1"),
+ new TimeRange[][] {
+ new TimeRange[] {new TimeRange(1, 100), new TimeRange(200, 300)},
+ new TimeRange[] {
+ new TimeRange(604799900, 604800020), new TimeRange(604810020,
604820020)
+ }
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4,
+ Arrays.asList(false, false));
+ writer.endChunkGroup();
+
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue(
+ Arrays.asList("s0", "s1"),
+ new TimeRange[][] {
+ new TimeRange[] {new TimeRange(1, 100), new TimeRange(200, 300)},
+ new TimeRange[] {
+ new TimeRange(604799900, 604800020), new TimeRange(604810020,
604820020)
+ }
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4,
+ Arrays.asList(false, false));
+ writer.endChunkGroup();
+
+ writer.endFile();
+ }
+ return resource;
+ }
+
private TsFileResource performCompaction()
throws StorageEngineException,
IOException,
