This is an automated email from the ASF dual-hosted git repository. yongzao pushed a commit to branch fix-getting-ttl-4-tree-model in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6356b45347544e65e5b2d35c8ba416a2c8befe76 Author: YongzaoDan <[email protected]> AuthorDate: Wed Apr 2 11:51:43 2025 +0800 Finish --- .../partition/IoTDBPartitionTableAutoCleanIT.java | 126 +++++++++++++++------ .../procedure/PartitionTableAutoCleaner.java | 5 + .../apache/iotdb/commons/schema/ttl/TTLCache.java | 23 +++- 3 files changed, 119 insertions(+), 35 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java index 6cad5508316..e41941cafea 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java @@ -27,6 +27,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.After; @@ -45,6 +46,9 @@ import java.util.concurrent.TimeUnit; @Category({ClusterIT.class}) public class IoTDBPartitionTableAutoCleanIT { + private static final String TREE_DATABASE_PREFIX = "root.db.g_"; + private static final String TABLE_DATABASE_PREFIX = "database_"; + private static final int TEST_REPLICATION_FACTOR = 1; private static final long TEST_TIME_PARTITION_INTERVAL = 604800000; private static final long TEST_TTL_CHECK_INTERVAL = 5_000; @@ -73,46 +77,100 @@ public class IoTDBPartitionTableAutoCleanIT { } @Test - public void testAutoCleanPartitionTable() throws Exception { - try (Connection connection = EnvFactory.getEnv().getConnection(); + public void testAutoCleanPartitionTableForTreeModel() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); Statement statement = connection.createStatement()) { - // Create db1 - statement.execute("CREATE DATABASE root.db1"); - statement.execute("CREATE TIMESERIES root.db1.s WITH DATATYPE=INT64,ENCODING=PLAIN"); - // Insert expired data - statement.execute( - String.format( - "INSERT INTO root.db1(timestamp, s) VALUES (%d, %d)", - TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1)); - // Insert existed data - statement.execute( - String.format( - "INSERT INTO root.db1(timestamp, s) VALUES (%d, %d)", - TEST_CURRENT_TIME_SLOT.getStartTime(), 1)); - // Let db.TTL > device.TTL, the valid TTL should be the bigger one - statement.execute("SET TTL TO root.db1 " + TEST_TTL); - statement.execute("SET TTL TO root.db1.s " + 10); - // Create db2 - statement.execute("CREATE DATABASE root.db2"); - statement.execute("CREATE TIMESERIES root.db2.s WITH DATATYPE=INT64,ENCODING=PLAIN"); - // Insert expired data + // Create databases and insert test data + for (int i = 0; i < 3; i++) { + String databaseName = String.format("%s%d", TREE_DATABASE_PREFIX, i); + statement.execute(String.format("CREATE DATABASE %s", databaseName)); + statement.execute( + String.format( + "CREATE TIMESERIES %s.s WITH DATATYPE=INT64,ENCODING=PLAIN", databaseName)); + // Insert expired data + statement.execute( + String.format( + "INSERT INTO %s(timestamp, s) VALUES (%d, %d)", + databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1)); + // Insert existed data + statement.execute( + String.format( + "INSERT INTO %s(timestamp, s) VALUES (%d, %d)", + databaseName, TEST_CURRENT_TIME_SLOT.getStartTime(), 1)); + } + // Let db0.TTL > device.TTL, the valid TTL should be the bigger one + statement.execute(String.format("SET TTL TO %s0 %d", TREE_DATABASE_PREFIX, TEST_TTL)); + statement.execute(String.format("SET TTL TO %s0.s %d", TREE_DATABASE_PREFIX, 10)); + // Let db1.TTL < device.TTL, the valid TTL should be the bigger one + statement.execute(String.format("SET TTL TO %s1 %d", TREE_DATABASE_PREFIX, 10)); + statement.execute(String.format("SET TTL TO %s1.s %d", TREE_DATABASE_PREFIX, TEST_TTL)); + // Set TTL to path db2.** + statement.execute(String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX, TEST_TTL)); + } + + TDataPartitionReq req = new TDataPartitionReq(); + for (int i = 0; i < 3; i++) { + req.putToPartitionSlotsMap(String.format("%s%d", TREE_DATABASE_PREFIX, i), new TreeMap<>()); + } + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + for (int retry = 0; retry < 120; retry++) { + boolean partitionTableAutoCleaned = true; + TDataPartitionTableResp resp = client.getDataPartitionTable(req); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) { + partitionTableAutoCleaned = + resp.getDataPartitionTable().entrySet().stream() + .flatMap(e1 -> e1.getValue().entrySet().stream()) + .allMatch(e2 -> e2.getValue().size() == 1); + } + if (partitionTableAutoCleaned) { + return; + } + TimeUnit.SECONDS.sleep(1); + } + } + Assert.fail("The PartitionTable in the ConfigNode is not auto cleaned!"); + } + + @Test + public void testAutoCleanPartitionTableForTableModel() throws Exception { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + // Create databases and insert test data + for (int i = 0; i < 2; i++) { + System.out.println(i); + String databaseName = String.format("%s%d", TABLE_DATABASE_PREFIX, i); + statement.execute(String.format("CREATE DATABASE IF NOT EXISTS %s", databaseName)); + statement.execute(String.format("USE DATABASE %s", databaseName)); + statement.execute("CREATE TABLE tb (time TIMESTAMP TIME, s int64 FIELD)"); + // Insert expired data + statement.execute( + String.format( + "INSERT INTO tb(timestamp, s) VALUES (%d, %d);", + TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1)); + // Insert existed data + statement.execute( + String.format( + "INSERT INTO tb(timestamp, s) VALUES (%d, %d);", + TEST_CURRENT_TIME_SLOT.getStartTime(), 1)); + } + // Let db0.TTL > table.TTL, the valid TTL should be the bigger one statement.execute( - String.format( - "INSERT INTO root.db2(timestamp, s) VALUES (%d, %d)", - TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1)); - // Insert existed data + String.format("ALTER DATABASE %s0 WITH(TTL=%d)", TABLE_DATABASE_PREFIX, TEST_TTL)); + statement.execute(String.format("USE DATABASE %s0", TABLE_DATABASE_PREFIX)); + statement.execute(String.format("ALTER TABLE tb SET PROPERTIES TTL=%d", 10)); + // Let db1.TTL < table.TTL, the valid TTL should be the bigger one statement.execute( - String.format( - "INSERT INTO root.db2(timestamp, s) VALUES (%d, %d)", - TEST_CURRENT_TIME_SLOT.getStartTime(), 1)); - // Let db.TTL < device.TTL, the valid TTL should be the bigger one - statement.execute("SET TTL TO root.db2 " + 10); - statement.execute("SET TTL TO root.db2.s " + TEST_TTL); + String.format("ALTER DATABASE %s1 WITH(TTL=%d)", TABLE_DATABASE_PREFIX, 10)); + statement.execute(String.format("USE DATABASE %s1", TABLE_DATABASE_PREFIX)); + statement.execute(String.format("ALTER TABLE tb SET PROPERTIES TTL=%d", TEST_TTL)); } TDataPartitionReq req = new TDataPartitionReq(); - req.putToPartitionSlotsMap("root.db1", new TreeMap<>()); - req.putToPartitionSlotsMap("root.db2", new TreeMap<>()); + for (int i = 0; i < 2; i++) { + req.putToPartitionSlotsMap(String.format("%s%d", TABLE_DATABASE_PREFIX, i), new TreeMap<>()); + } try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { for (int retry = 0; retry < 120; retry++) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java index 3c9690d3e29..2357363ef04 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java @@ -63,6 +63,11 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> { ? configManager.getClusterSchemaManager().getDatabaseMaxTTL(database) : configManager.getTTLManager().getDatabaseMaxTTL(database); databaseTTLMap.put(database, databaseTTL); + LOGGER.info( + "[PartitionTableCleaner] Database: {} is TableModel: {}, TTL: {}", + database, + PathUtils.isTableModelDatabase(database), + databaseTTL); if (!configManager.getPartitionManager().isDatabaseExist(database) || databaseTTL < 0 || databaseTTL == Long.MAX_VALUE) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java index 0adc61ca9af..bfb67069ccc 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java @@ -182,7 +182,7 @@ public class TTLCache { * TTL is not set or the database does not exist. */ public long getDatabaseMaxTTL(String database) { - CacheNode node = ttlCacheTree.getChild(database); + CacheNode node = ttlCacheTree.searchChild(database); if (node == null) { return NULL_TTL; } @@ -300,6 +300,27 @@ public class TTLCache { return children.get(name); } + /** + * Search the child node by name. + * + * @param name the name corresponding to the child node, use '.' to separate each node + * @return the child node if it exists, otherwise return null + */ + public CacheNode searchChild(String name) { + String[] nodeNames = name.split("\\."); + CacheNode current = this; + for (String nodeName : nodeNames) { + if (nodeName.equals("root")) { + continue; + } + current = current.getChild(nodeName); + if (current == null) { + return null; + } + } + return current; + } + public Map<String, CacheNode> getChildren() { return children; }
