OneSizeFitsQuorum commented on code in PR #14737:
URL: https://github.com/apache/iotdb/pull/14737#discussion_r1924669303
##########
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java:
##########
@@ -90,6 +90,8 @@ public enum ThreadName {
CONFIG_NODE_LOAD_PUBLISHER("Cluster-LoadStatistics-Publisher"),
// -------------------------- ConfigNode-RegionManagement
--------------------------
CONFIG_NODE_REGION_MAINTAINER("IoTDB-Region-Maintainer"),
+ // -------------------------- ConfigNode-PartitionManagement
--------------------------
+ CONFIG_NODE_PARTITION_CLEANER("IoTDB-Partition-Cleaner"),
Review Comment:
remove this thread?
##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java:
##########
@@ -1448,6 +1464,54 @@ public void stopRegionCleaner() {
}
}
+ /**
+ * The ConfigNode-leader will periodically invoke this interface to
automatically clean expired
+ * partition table.
+ */
+ public void autoCleanDataPartitionTable() {
+ List<String> databases = getClusterSchemaManager().getDatabaseNames(null);
+ Map<String, Long> databaseTTLMap =
getClusterSchemaManager().getTTLInfoForUpgrading();
+ for (String database : databases) {
+ long subTreeMaxTTL = getTTLManager().getDatabaseMaxTTL(database);
+ databaseTTLMap.put(
+ database, Math.max(subTreeMaxTTL,
databaseTTLMap.getOrDefault(database, -1L)));
Review Comment:
add some judgement like "isDatabaseExisted(database) && 0 < ttl && ttl <
Long.MAX_VALUE" here to remove overhead?
BTW, If all the databases don't have ttl, we can logically just do this
check and find that none of them need to be cleaned up, so there's no need to
do a consensus write
##########
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java:
##########
@@ -136,21 +142,31 @@ public class PartitionManager {
private static final String CONSENSUS_WRITE_ERROR =
"Failed in the write API executing the consensus layer due to: ";
- /** Region cleaner. */
// Monitor for leadership change
private final Object scheduleMonitor = new Object();
+ /** Region cleaner. */
// Try to delete Regions in every 10s
private static final int REGION_MAINTAINER_WORK_INTERVAL = 10;
+
private final ScheduledExecutorService regionMaintainer;
private Future<?> currentRegionMaintainerFuture;
+ /** Partition cleaner. */
+ private static final long PARTITION_CLEANER_WORK_INTERVAL =
COMMON_CONFIG.getTTLCheckInterval();
+
+ private final ScheduledExecutorService partitionCleaner;
+ private Future<?> currentPartitionCleanerFuture;
+
public PartitionManager(IManager configManager, PartitionInfo partitionInfo)
{
this.configManager = configManager;
this.partitionInfo = partitionInfo;
this.regionMaintainer =
IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor(
ThreadName.CONFIG_NODE_REGION_MAINTAINER.getName());
+ this.partitionCleaner =
Review Comment:
maybe try to reuse procedure periodic tasks
##########
integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanTest.java:
##########
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it.partition;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
+import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.utils.TimePartitionUtils;
+import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
+import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({ClusterIT.class})
+public class IoTDBPartitionTableAutoCleanTest {
+
+ private static final int TEST_REPLICATION_FACTOR = 1;
+ private static final long TEST_TIME_PARTITION_INTERVAL = 604800000;
+ private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
+ // private static final int TEST_SERIES_SLOT_NUM = 1000;
+ // private static final String TEST_SERIES_EXECUTOR_CLASS =
+ // "org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor";
+
+ // private static final SeriesPartitionExecutor TEST_EXECUTOR =
+ //
SeriesPartitionExecutor.getSeriesPartitionExecutor(TEST_SERIES_EXECUTOR_CLASS,
+ // TEST_SERIES_SLOT_NUM);
+ private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT =
+ TimePartitionUtils.getCurrentTimePartitionSlot();
+ private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL;
+
+ @Before
+ public void setUp() throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ // .setSeriesSlotNum(TEST_SERIES_SLOT_NUM)
Review Comment:
please do not comment code
##########
integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanTest.java:
##########
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it.partition;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
+import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.utils.TimePartitionUtils;
+import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
+import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({ClusterIT.class})
+public class IoTDBPartitionTableAutoCleanTest {
+
+ private static final int TEST_REPLICATION_FACTOR = 1;
+ private static final long TEST_TIME_PARTITION_INTERVAL = 604800000;
+ private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
+ // private static final int TEST_SERIES_SLOT_NUM = 1000;
+ // private static final String TEST_SERIES_EXECUTOR_CLASS =
Review Comment:
please do not comment code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
