This is an automated email from the ASF dual-hosted git repository.
HTHou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 881ed81484b Fix deletion matching during compaction (#18732)
881ed81484b is described below
commit 881ed81484b749bbcffcf0fbe4d0b6cfde77154a
Author: Haonan <[email protected]>
AuthorDate: Mon Sep 28 15:04:06 2026 +0800
Fix deletion matching during compaction (#18732)
---
.../it/IoTDBRecreatedTimeseriesCompactionIT.java | 125 +++++++++++++++++++++
.../execute/utils/CompactionPathUtils.java | 2 +-
.../compaction/utils/CompactionUtilsTest.java | 43 +++++--
3 files changed, 161 insertions(+), 9 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java
new file mode 100644
index 00000000000..8d3609ee872
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecreatedTimeseriesCompactionIT.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+
+import org.awaitility.Awaitility;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+import static org.apache.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({LocalStandaloneIT.class})
+public class IoTDBRecreatedTimeseriesCompactionIT {
+
+ private static final String DATABASE = "root.repro";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setEnableSeqSpaceCompaction(false)
+ .setEnableUnseqSpaceCompaction(false)
+ .setEnableCrossSpaceCompaction(false)
+ .setInnerCompactionCandidateFileNum(2);
+
EnvFactory.getEnv().getConfig().getDataNodeConfig().setCompactionScheduleInterval(1000);
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void
testDeletionAfterRecreatingTwoLevelTimeseriesSurvivesCompaction() throws
Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + DATABASE);
+ statement.execute("CREATE TIMESERIES root.repro.s1 WITH DATATYPE=INT64");
+ statement.execute("INSERT INTO root.repro(time, s1) VALUES (1, 1)");
+ statement.execute("FLUSH");
+
+ statement.execute("DELETE TIMESERIES root.repro.s1");
+ statement.execute("CREATE TIMESERIES root.repro.s1 WITH DATATYPE=INT64");
+ statement.execute("INSERT INTO root.repro(time, s1) VALUES (2, 2)");
+ statement.execute("FLUSH");
+
+ assertOnlyRecreatedTimeseriesDataExists(statement);
+ Assert.assertEquals(2, countSequenceTsFiles());
+
+ statement.execute("SET CONFIGURATION
'enable_seq_space_compaction'='true'");
+
+ Awaitility.await()
+ .pollInterval(200, TimeUnit.MILLISECONDS)
+ .atMost(30, TimeUnit.SECONDS)
+ .until(() -> countSequenceTsFiles() == 1);
+
+ assertOnlyRecreatedTimeseriesDataExists(statement);
+ }
+ }
+
+ private static void assertOnlyRecreatedTimeseriesDataExists(Statement
statement)
+ throws Exception {
+ try (ResultSet resultSet = statement.executeQuery("SELECT s1 FROM
root.repro")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(2, resultSet.getLong("Time"));
+ Assert.assertEquals(2, resultSet.getLong("root.repro.s1"));
+ Assert.assertFalse(resultSet.next());
+ }
+ }
+
+ private static long countSequenceTsFiles() throws IOException {
+ Path databaseSequenceDirectory =
+ Paths.get(
+ EnvFactory.getEnv().getDataNodeWrapper(0).getDataPath(),
+ IoTDBConstant.SEQUENCE_FOLDER_NAME,
+ DATABASE);
+ if (!Files.exists(databaseSequenceDirectory)) {
+ return 0;
+ }
+ try (Stream<Path> paths = Files.walk(databaseSequenceDirectory)) {
+ return paths
+ .filter(Files::isRegularFile)
+ .filter(path ->
path.getFileName().toString().endsWith(TSFILE_SUFFIX))
+ .count();
+ }
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java
index 0b26202a0ac..898ea1729f0 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java
@@ -38,7 +38,7 @@ public class CompactionPathUtils {
public static PartialPath getPath(final IDeviceID device) throws
IllegalPathException {
if (device.isTableModel()) {
- final String[] nodes = new String[device.segmentNum() + 1];
+ final String[] nodes = new String[device.segmentNum()];
nodes[0] = device.getTableName();
for (int i = 0; i < device.segmentNum() - 1; i++) {
nodes[i + 1] = device.segment(i + 1) == null ? null : device.segment(i
+ 1).toString();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java
index 97427b16321..2a0890e6277 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java
@@ -21,14 +21,16 @@ package
org.apache.iotdb.db.storageengine.dataregion.compaction.utils;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.path.MeasurementPath;
-import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PatternTreeMap;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionPathUtils;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
+import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import
org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
+import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -38,6 +40,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
public class CompactionUtilsTest extends AbstractCompactionTest {
@Override
@@ -52,13 +55,37 @@ public class CompactionUtilsTest extends
AbstractCompactionTest {
}
@Test
- public void testCompactionPathUtils() {
- try {
- IDeviceID deviceID = new StringArrayDeviceID(new String[] {"db.table1",
null, "tag1"});
- PartialPath path = CompactionPathUtils.getPath(deviceID, "s1");
- } catch (Exception e) {
- Assert.fail();
- }
+ public void testCompactionPathUtilsForTableModelDevice() throws Exception {
+ IDeviceID deviceID = new StringArrayDeviceID(new String[] {"db.table1",
null, "tag1"});
+
+ Assert.assertArrayEquals(
+ new String[] {"db.table1", null, "tag1"},
CompactionPathUtils.getPath(deviceID).getNodes());
+ Assert.assertArrayEquals(
+ new String[] {"db.table1", null, "tag1", "s1"},
+ CompactionPathUtils.getPath(deviceID, "s1").getNodes());
+ }
+
+ @Test
+ public void testCompactionPathUtilsForTwoLevelTreeDevice() throws Exception {
+ MeasurementPath deletionPath = new MeasurementPath("root.repro.s1");
+ IDeviceID deviceID = deletionPath.getIDeviceID();
+
+ Assert.assertArrayEquals(
+ new String[] {"root", "repro"},
CompactionPathUtils.getPath(deviceID).getNodes());
+ Assert.assertArrayEquals(
+ deletionPath.getNodes(), CompactionPathUtils.getPath(deviceID,
"s1").getNodes());
+
+ TreeDeletionEntry deletion =
+ new TreeDeletionEntry(deletionPath, Long.MIN_VALUE, Long.MAX_VALUE);
+ PatternTreeMap<ModEntry, PatternTreeMapFactory.ModsSerializer>
modifications =
+ PatternTreeMapFactory.getModsPatternTreeMap();
+ modifications.append(deletion.keyOfPatternTree(), deletion);
+
+ List<ModEntry> matchedModifications =
+ CompactionUtils.getMatchedModifications(modifications, deviceID, "s1",
null);
+ Assert.assertEquals(1, matchedModifications.size());
+ Assert.assertEquals(
+ deletionPath, ((TreeDeletionEntry)
matchedModifications.get(0)).getPathPattern());
}
@Test
