This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 31a407ac402 Skip TTL check when there is no TTL and no mods file in
the data region (#16110) (#16129)
31a407ac402 is described below
commit 31a407ac402987089d32effd3534ad94a92878f7
Author: shuwenwei <[email protected]>
AuthorDate: Tue Aug 12 15:39:05 2025 +0800
Skip TTL check when there is no TTL and no mods file in the data region
(#16110) (#16129)
* Skip TTL check when there is no ttl in the data region and no mods file
* check back quote
---
.../iotdb/confignode/persistence/TTLInfo.java | 2 +
.../analyze/cache/schema/DataNodeTTLCache.java | 9 ++++
.../db/storageengine/dataregion/DataRegion.java | 27 ++++++++++-
.../iotdb/db/storageengine/dataregion/TTLTest.java | 52 ++++++++++++++++++++
.../apache/iotdb/commons/schema/ttl/TTLCache.java | 56 ++++++++++++++++++++--
5 files changed, 142 insertions(+), 4 deletions(-)
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java
index 131838b6ef6..9587730988c 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java
@@ -168,6 +168,8 @@ public class TTLInfo implements SnapshotProcessor {
lock.readLock().lock();
try {
return ttlCache.getDatabaseMaxTTL(database);
+ } catch (IllegalPathException e) {
+ return TTLCache.NULL_TTL;
} finally {
lock.readLock().unlock();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
index 5c762305159..095a78b9b2c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
@@ -122,6 +122,15 @@ public class DataNodeTTLCache {
}
}
+ public boolean dataInDatabaseMayHaveTTL(String db) throws
IllegalPathException {
+ lock.readLock().lock();
+ try {
+ return ttlCache.dataInDatabaseMayHaveTTL(db);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
/**
* Get ttl of one specific path node without time precision conversion. If
this node does not set
* ttl, then return -1.
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 307ff057e1d..371807e9dd1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -174,6 +174,7 @@ import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
import static
org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet.SEQUENCE_TSFILE;
@@ -2749,9 +2750,12 @@ public class DataRegion implements IDataRegionForQuery {
// wait until success
Thread.sleep(500);
}
- logger.info("[TTL] {}-{} Start ttl checking.", databaseName, dataRegionId);
int trySubmitCount = 0;
try {
+ if (skipCurrentTTLAndModificationCheck()) {
+ return 0;
+ }
+ logger.info("[TTL] {}-{} Start ttl and modification checking.",
databaseName, dataRegionId);
CompactionScheduleContext context = new CompactionScheduleContext();
List<Long> timePartitions = new
ArrayList<>(tsFileManager.getTimePartitions());
// Sort the time partition from smallest to largest
@@ -2787,6 +2791,27 @@ public class DataRegion implements IDataRegionForQuery {
return trySubmitCount;
}
+ private boolean skipCurrentTTLAndModificationCheck() {
+ if (this.databaseName.equals(SchemaConstant.SYSTEM_DATABASE)) {
+ return true;
+ }
+ for (Long timePartition : getTimePartitions()) {
+ List<TsFileResource> seqFiles =
tsFileManager.getTsFileListSnapshot(timePartition, true);
+ List<TsFileResource> unseqFiles =
tsFileManager.getTsFileListSnapshot(timePartition, false);
+ boolean modFileExists =
+ Stream.concat(seqFiles.stream(), unseqFiles.stream())
+ .anyMatch(TsFileResource::modFileExists);
+ if (modFileExists) {
+ return false;
+ }
+ }
+ try {
+ return
!DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL(databaseName);
+ } catch (Exception ignored) {
+ return false;
+ }
+ }
+
protected int[] executeInsertionCompaction(
List<Long> timePartitions, CompactionScheduleContext context) throws
InterruptedException {
int[] trySubmitCountOfTimePartitions = new int[timePartitions.size()];
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java
index 9e5920f1ea9..a2804de4128 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java
@@ -59,6 +59,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.common.block.TsBlock;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -257,6 +258,57 @@ public class TTLTest {
assertTrue(cnt == 0);
}
+ @Test
+ public void testTTLRead2() throws IllegalPathException {
+ DataNodeTTLCache.getInstance().setTTL("root.test1.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.test1.d1.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.test2.**", 500);
+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.test2.d1.**", 500);
+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.test.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.test1.**", 500);
+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.test.sg1.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.test.sg2.**", 500);
+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.test.sg1.d1.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+
+ DataNodeTTLCache.getInstance().setTTL("root.`1.1`.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.`1.1`"));
+ DataNodeTTLCache.getInstance().clearAllTTL();
+ }
+
private MeasurementPath mockMeasurementPath() throws MetadataException {
return new MeasurementPath(
new PartialPath(sg1 + TsFileConstant.PATH_SEPARATOR + s1),
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java
index de9ef054a64..9334a7ad2b4 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.StatusUtils;
import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import javax.annotation.concurrent.NotThreadSafe;
@@ -177,7 +178,7 @@ public class TTLCache {
* @return the maximum ttl of the subtree of the corresponding database.
return NULL_TTL if the
* TTL is not set or the database does not exist.
*/
- public long getDatabaseMaxTTL(String database) {
+ public long getDatabaseMaxTTL(String database) throws IllegalPathException {
CacheNode node = ttlCacheTree.searchChild(database);
if (node == null) {
return NULL_TTL;
@@ -195,6 +196,55 @@ public class TTLCache {
return maxTTL;
}
+ public boolean dataInDatabaseMayHaveTTL(String database) throws
IllegalPathException {
+ String[] nodeNames = split(database);
+ CacheNode current = ttlCacheTree;
+ for (String nodeName : nodeNames) {
+ if (hasValidTTLOnCurrentLevel(current)) {
+ return true;
+ }
+ if (nodeName.equals("root")) {
+ continue;
+ }
+ current = current.getChild(nodeName);
+ if (current == null) {
+ return false;
+ }
+ }
+
+ if (hasValidTTLOnCurrentLevel(current)) {
+ return true;
+ }
+
+ Queue<CacheNode> queue = new LinkedList<>();
+ queue.add(current);
+ while (!queue.isEmpty()) {
+ current = queue.poll();
+ for (CacheNode child : current.getChildren().values()) {
+ if (child.ttl >= 0 && child.ttl != Long.MAX_VALUE) {
+ return true;
+ }
+ queue.add(child);
+ }
+ }
+ return false;
+ }
+
+ private boolean hasValidTTLOnCurrentLevel(CacheNode current) {
+ if (current.ttl >= 0 && current.ttl != Long.MAX_VALUE) {
+ return true;
+ }
+ CacheNode wildcardChild =
current.getChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD);
+ return wildcardChild != null && wildcardChild.ttl >= 0 &&
wildcardChild.ttl != Long.MAX_VALUE;
+ }
+
+ private static String[] split(String path) throws IllegalPathException {
+ if (!path.contains(TsFileConstant.BACK_QUOTE_STRING)) {
+ return path.split(TsFileConstant.PATH_SEPARATER_NO_REGEX);
+ }
+ return PathUtils.splitPathToDetachedNodes(path);
+ }
+
/**
* @return key is path contains wildcard between each node
*/
@@ -302,8 +352,8 @@ public class TTLCache {
* @param name the name corresponding to the child node, use '.' to
separate each node
* @return the child node if it exists, otherwise return null
*/
- public CacheNode searchChild(String name) {
- String[] nodeNames = name.split("\\.");
+ public CacheNode searchChild(String name) throws IllegalPathException {
+ String[] nodeNames = split(name);
CacheNode current = this;
for (String nodeName : nodeNames) {
if (nodeName.equals("root")) {
