This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch rc/1.3.4-1 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4e62af2c26c2eb6918848e597c14478eb6df9126 Author: Yongzao <[email protected]> AuthorDate: Tue May 13 17:07:53 2025 +0800 [To dev/1.3] Use reference time position for PartitionTableAutoCleaner --- .../partition/IoTDBPartitionTableAutoCleanIT.java | 7 +++-- ....java => IoTDBPartitionTableAutoCleanUSIT.java} | 30 +++++++++++++--------- .../iotdb/confignode/manager/TTLManager.java | 9 +++++-- .../procedure/PartitionTableAutoCleaner.java | 29 ++++++++++++++++++--- .../iotdb/commons/utils/TimePartitionUtils.java | 13 ---------- 5 files changed, 56 insertions(+), 32 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java index c62739cdc13..a7addb87c1d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java @@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.it.partition; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; import org.apache.iotdb.it.env.EnvFactory; @@ -52,7 +51,11 @@ public class IoTDBPartitionTableAutoCleanIT { private static final long TEST_TTL_CHECK_INTERVAL = 5_000; private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT = - TimePartitionUtils.getCurrentTimePartitionSlot(); + new TTimePartitionSlot() + .setStartTime( + System.currentTimeMillis() + / TEST_TIME_PARTITION_INTERVAL + * TEST_TIME_PARTITION_INTERVAL); private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL; @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java similarity index 84% copy from integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java copy to integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java index c62739cdc13..f8e94c28dcb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java @@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.it.partition; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; import org.apache.iotdb.it.env.EnvFactory; @@ -43,17 +42,22 @@ import java.util.concurrent.TimeUnit; @RunWith(IoTDBTestRunner.class) @Category({ClusterIT.class}) -public class IoTDBPartitionTableAutoCleanIT { +public class IoTDBPartitionTableAutoCleanUSIT { private static final String TREE_DATABASE_PREFIX = "root.db.g_"; private static final int TEST_REPLICATION_FACTOR = 1; - private static final long TEST_TIME_PARTITION_INTERVAL = 604800000; + private static final long TEST_TIME_PARTITION_INTERVAL_IN_MS = 604800_000; private static final long TEST_TTL_CHECK_INTERVAL = 5_000; private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT = - TimePartitionUtils.getCurrentTimePartitionSlot(); - private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL; + new TTimePartitionSlot() + .setStartTime( + System.currentTimeMillis() + * 1000L + / TEST_TIME_PARTITION_INTERVAL_IN_MS + * TEST_TIME_PARTITION_INTERVAL_IN_MS); + private static final long TEST_TTL_IN_MS = 7 * TEST_TIME_PARTITION_INTERVAL_IN_MS; @Before public void setUp() throws Exception { @@ -62,8 +66,10 @@ public class IoTDBPartitionTableAutoCleanIT { .getCommonConfig() .setSchemaReplicationFactor(TEST_REPLICATION_FACTOR) .setDataReplicationFactor(TEST_REPLICATION_FACTOR) - .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL) - .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL); + .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL_IN_MS) + .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL) + // Note that the time precision of IoTDB is us in this IT + .setTimestampPrecision("us"); // Init 1C1D environment EnvFactory.getEnv().initClusterEnvironment(1, 1); @@ -89,7 +95,7 @@ public class IoTDBPartitionTableAutoCleanIT { statement.execute( String.format( "INSERT INTO %s(timestamp, s) VALUES (%d, %d)", - databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1)); + databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL_IN_MS * 2000, -1)); // Insert existed data statement.execute( String.format( @@ -97,15 +103,15 @@ public class IoTDBPartitionTableAutoCleanIT { databaseName, TEST_CURRENT_TIME_SLOT.getStartTime(), 1)); } // Let db0.TTL > device.TTL, the valid TTL should be the bigger one - statement.execute(String.format("SET TTL TO %s0 %d", TREE_DATABASE_PREFIX, TEST_TTL)); + statement.execute(String.format("SET TTL TO %s0 %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS)); statement.execute(String.format("SET TTL TO %s0.s %d", TREE_DATABASE_PREFIX, 10)); // Let db1.TTL < device.TTL, the valid TTL should be the bigger one statement.execute(String.format("SET TTL TO %s1 %d", TREE_DATABASE_PREFIX, 10)); - statement.execute(String.format("SET TTL TO %s1.s %d", TREE_DATABASE_PREFIX, TEST_TTL)); + statement.execute(String.format("SET TTL TO %s1.s %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS)); // Set TTL to path db2.** - statement.execute(String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX, TEST_TTL)); + statement.execute( + String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS)); } - TDataPartitionReq req = new TDataPartitionReq(); for (int i = 0; i < 3; i++) { req.putToPartitionSlotsMap(String.format("%s%d", TREE_DATABASE_PREFIX, i), new TreeMap<>()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java index 55a61389126..b0c4dd5f296 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.confignode.consensus.request.read.ttl.ShowTTLPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; @@ -135,8 +136,12 @@ public class TTLManager { * @return the maximum ttl of the subtree of the corresponding database. return NULL_TTL if the * TTL is not set or the database does not exist. */ - public long getDatabaseMaxTTL(String database) { - return ttlInfo.getDatabaseMaxTTL(database); + public long getDatabaseMaxTTL(final String database) { + final long ttl = ttlInfo.getDatabaseMaxTTL(database); + return ttl == Long.MAX_VALUE || ttl < 0 + ? ttl + : CommonDateTimeUtils.convertMilliTimeWithPrecision( + ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision()); } /** Only used for upgrading from old database-level ttl to device-level ttl. */ diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java index ce5d07276db..8a08e5efe99 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java @@ -22,7 +22,6 @@ package org.apache.iotdb.confignode.procedure; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.consensus.request.write.partition.AutoCleanPartitionTablePlan; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -42,6 +41,10 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> { private static final Logger LOGGER = LoggerFactory.getLogger(PartitionTableAutoCleaner.class); private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); + + private static final String timestampPrecision = + CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + private final ConfigManager configManager; public PartitionTableAutoCleaner(ConfigManager configManager) { @@ -59,6 +62,12 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> { for (String database : databases) { long databaseTTL = configManager.getTTLManager().getDatabaseMaxTTL(database); databaseTTLMap.put(database, databaseTTL); + } + LOGGER.info( + "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner, databaseTTL: {}", + databaseTTLMap); + for (String database : databases) { + long databaseTTL = databaseTTLMap.get(database); if (!configManager.getPartitionManager().isDatabaseExist(database) || databaseTTL < 0 || databaseTTL == Long.MAX_VALUE) { @@ -71,8 +80,7 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> { "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner for: {}", databaseTTLMap); // Only clean the partition table when necessary - TTimePartitionSlot currentTimePartitionSlot = - TimePartitionUtils.getCurrentTimePartitionSlot(); + TTimePartitionSlot currentTimePartitionSlot = getCurrentTimePartitionSlot(); try { configManager .getConsensusManager() @@ -82,4 +90,19 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> { } } } + + /** + * @return The time partition slot corresponding to current timestamp. Note that we do not shift + * the start time to the correct starting point, since this interface only constructs a time + * reference position for the partition table cleaner. + */ + private static TTimePartitionSlot getCurrentTimePartitionSlot() { + if ("ms".equals(timestampPrecision)) { + return new TTimePartitionSlot(System.currentTimeMillis()); + } else if ("us".equals(timestampPrecision)) { + return new TTimePartitionSlot(System.currentTimeMillis() * 1000); + } else { + return new TTimePartitionSlot(System.currentTimeMillis() * 1000_000); + } + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java index ed01e8a5f3f..7b331fddaac 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java @@ -34,9 +34,6 @@ public class TimePartitionUtils { private static long timePartitionOrigin = CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin(); - private static String timestampPrecision = - CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); - /** Time range for dividing database, the time unit is the same with IoTDB's TimestampPrecision */ private static long timePartitionInterval = CommonDescriptor.getInstance().getConfig().getTimePartitionInterval(); @@ -74,16 +71,6 @@ public class TimePartitionUtils { } } - public static TTimePartitionSlot getCurrentTimePartitionSlot() { - if ("ms".equals(timestampPrecision)) { - return getTimePartitionSlot(System.currentTimeMillis()); - } else if ("us".equals(timestampPrecision)) { - return getTimePartitionSlot(System.currentTimeMillis() * 1000); - } else { - return getTimePartitionSlot(System.currentTimeMillis() * 1000_000); - } - } - public static TTimePartitionSlot getTimePartitionSlot(long time) { TTimePartitionSlot timePartitionSlot = new TTimePartitionSlot(); timePartitionSlot.setStartTime(getTimePartitionLowerBound(time));
