This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch skipTTLCheckInSomeCases-1.3 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 380ea0922bf5532372d6f443342794bdaa2a83b1 Author: shuwenwei <[email protected]> AuthorDate: Thu Aug 7 10:41:10 2025 +0800 Skip TTL check when there is no TTL and no mods file in the data region (#16110) * Skip TTL check when there is no ttl in the data region and no mods file * check back quote --- .../iotdb/confignode/persistence/TTLInfo.java | 2 + .../analyze/cache/schema/DataNodeTTLCache.java | 9 ++++ .../db/storageengine/dataregion/DataRegion.java | 27 ++++++++++- .../iotdb/db/storageengine/dataregion/TTLTest.java | 52 ++++++++++++++++++++ .../apache/iotdb/commons/schema/ttl/TTLCache.java | 56 ++++++++++++++++++++-- 5 files changed, 142 insertions(+), 4 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java index 131838b6ef6..9587730988c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java @@ -168,6 +168,8 @@ public class TTLInfo implements SnapshotProcessor { lock.readLock().lock(); try { return ttlCache.getDatabaseMaxTTL(database); + } catch (IllegalPathException e) { + return TTLCache.NULL_TTL; } finally { lock.readLock().unlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java index 5c762305159..095a78b9b2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java @@ -122,6 +122,15 @@ public class DataNodeTTLCache { } } + public boolean dataInDatabaseMayHaveTTL(String db) throws IllegalPathException { + lock.readLock().lock(); + try { + return ttlCache.dataInDatabaseMayHaveTTL(db); + } finally { + lock.readLock().unlock(); + } + } + /** * Get ttl of one specific path node without time precision conversion. If this node does not set * ttl, then return -1. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 307ff057e1d..371807e9dd1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -174,6 +174,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR; import static org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet.SEQUENCE_TSFILE; @@ -2749,9 +2750,12 @@ public class DataRegion implements IDataRegionForQuery { // wait until success Thread.sleep(500); } - logger.info("[TTL] {}-{} Start ttl checking.", databaseName, dataRegionId); int trySubmitCount = 0; try { + if (skipCurrentTTLAndModificationCheck()) { + return 0; + } + logger.info("[TTL] {}-{} Start ttl and modification checking.", databaseName, dataRegionId); CompactionScheduleContext context = new CompactionScheduleContext(); List<Long> timePartitions = new ArrayList<>(tsFileManager.getTimePartitions()); // Sort the time partition from smallest to largest @@ -2787,6 +2791,27 @@ public class DataRegion implements IDataRegionForQuery { return trySubmitCount; } + private boolean skipCurrentTTLAndModificationCheck() { + if (this.databaseName.equals(SchemaConstant.SYSTEM_DATABASE)) { + return true; + } + for (Long timePartition : getTimePartitions()) { + List<TsFileResource> seqFiles = tsFileManager.getTsFileListSnapshot(timePartition, true); + List<TsFileResource> unseqFiles = tsFileManager.getTsFileListSnapshot(timePartition, false); + boolean modFileExists = + Stream.concat(seqFiles.stream(), unseqFiles.stream()) + .anyMatch(TsFileResource::modFileExists); + if (modFileExists) { + return false; + } + } + try { + return !DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL(databaseName); + } catch (Exception ignored) { + return false; + } + } + protected int[] executeInsertionCompaction( List<Long> timePartitions, CompactionScheduleContext context) throws InterruptedException { int[] trySubmitCountOfTimePartitions = new int[timePartitions.size()]; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java index 9e5920f1ea9..a2804de4128 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java @@ -59,6 +59,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -257,6 +258,57 @@ public class TTLTest { assertTrue(cnt == 0); } + @Test + public void testTTLRead2() throws IllegalPathException { + DataNodeTTLCache.getInstance().setTTL("root.test1.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.test1.d1.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.test2.**", 500); + Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.test2.d1.**", 500); + Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.test.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.test1.**", 500); + Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.test.sg1.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.test.sg2.**", 500); + Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.test.sg1.d1.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTL(); + + DataNodeTTLCache.getInstance().setTTL("root.`1.1`.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.`1.1`")); + DataNodeTTLCache.getInstance().clearAllTTL(); + } + private MeasurementPath mockMeasurementPath() throws MetadataException { return new MeasurementPath( new PartialPath(sg1 + TsFileConstant.PATH_SEPARATOR + s1), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java index de9ef054a64..9334a7ad2b4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.utils.ReadWriteIOUtils; import javax.annotation.concurrent.NotThreadSafe; @@ -177,7 +178,7 @@ public class TTLCache { * @return the maximum ttl of the subtree of the corresponding database. return NULL_TTL if the * TTL is not set or the database does not exist. */ - public long getDatabaseMaxTTL(String database) { + public long getDatabaseMaxTTL(String database) throws IllegalPathException { CacheNode node = ttlCacheTree.searchChild(database); if (node == null) { return NULL_TTL; @@ -195,6 +196,55 @@ public class TTLCache { return maxTTL; } + public boolean dataInDatabaseMayHaveTTL(String database) throws IllegalPathException { + String[] nodeNames = split(database); + CacheNode current = ttlCacheTree; + for (String nodeName : nodeNames) { + if (hasValidTTLOnCurrentLevel(current)) { + return true; + } + if (nodeName.equals("root")) { + continue; + } + current = current.getChild(nodeName); + if (current == null) { + return false; + } + } + + if (hasValidTTLOnCurrentLevel(current)) { + return true; + } + + Queue<CacheNode> queue = new LinkedList<>(); + queue.add(current); + while (!queue.isEmpty()) { + current = queue.poll(); + for (CacheNode child : current.getChildren().values()) { + if (child.ttl >= 0 && child.ttl != Long.MAX_VALUE) { + return true; + } + queue.add(child); + } + } + return false; + } + + private boolean hasValidTTLOnCurrentLevel(CacheNode current) { + if (current.ttl >= 0 && current.ttl != Long.MAX_VALUE) { + return true; + } + CacheNode wildcardChild = current.getChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD); + return wildcardChild != null && wildcardChild.ttl >= 0 && wildcardChild.ttl != Long.MAX_VALUE; + } + + private static String[] split(String path) throws IllegalPathException { + if (!path.contains(TsFileConstant.BACK_QUOTE_STRING)) { + return path.split(TsFileConstant.PATH_SEPARATER_NO_REGEX); + } + return PathUtils.splitPathToDetachedNodes(path); + } + /** * @return key is path contains wildcard between each node */ @@ -302,8 +352,8 @@ public class TTLCache { * @param name the name corresponding to the child node, use '.' to separate each node * @return the child node if it exists, otherwise return null */ - public CacheNode searchChild(String name) { - String[] nodeNames = name.split("\\."); + public CacheNode searchChild(String name) throws IllegalPathException { + String[] nodeNames = split(name); CacheNode current = this; for (String nodeName : nodeNames) { if (nodeName.equals("root")) {
