This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 3cfe04692c1 RatisConsensus: extend election timeout threshold (#15844)
3cfe04692c1 is described below
commit 3cfe04692c1c3986e518038674203bedaf77ecf4
Author: GewuNewOne <[email protected]>
AuthorDate: Mon Aug 18 14:15:51 2025 +0800
RatisConsensus: extend election timeout threshold (#15844)
* RatisConsensus: modify election timeout threshold from 10s to 30s
* RatisConsensus: add a parameter named 'ratis_transfer_leader_timeout_ms'
to modify election timeout threshold
* Return 'this' to use set_function in chained calls.
---
.../iotdb/confignode/conf/ConfigNodeConfig.java | 10 ++++++++
.../confignode/conf/ConfigNodeDescriptor.java | 6 +++++
.../manager/consensus/ConsensusManager.java | 5 ++++
.../apache/iotdb/consensus/config/RatisConfig.java | 27 ++++++++++++++++++----
.../exception/RatisRequestFailedException.java | 9 ++++++++
.../iotdb/consensus/ratis/RatisConsensus.java | 23 ++++++++++++++----
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 ++
.../db/consensus/DataRegionConsensusImpl.java | 4 ++++
.../db/consensus/SchemaRegionConsensusImpl.java | 5 ++++
.../conf/iotdb-system.properties.template | 4 ++++
.../src/main/thrift/confignode.thrift | 2 ++
12 files changed, 98 insertions(+), 9 deletions(-)
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 50c8f5bc9a0..868949c0b22 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -294,6 +294,8 @@ public class ConfigNodeConfig {
private long ratisFirstElectionTimeoutMinMs = 50;
private long ratisFirstElectionTimeoutMaxMs = 150;
+ private int ratisTransferLeaderTimeoutMs = 30 * 1000; // 30s
+
private long configNodeRatisLogMax = 2L * 1024 * 1024 * 1024; // 2G
private long schemaRegionRatisLogMax = 2L * 1024 * 1024 * 1024; // 2G
private long dataRegionRatisLogMax = 20L * 1024 * 1024 * 1024; // 20G
@@ -1106,6 +1108,14 @@ public class ConfigNodeConfig {
this.ratisFirstElectionTimeoutMaxMs = ratisFirstElectionTimeoutMaxMs;
}
+ public int getRatisTransferLeaderTimeoutMs() {
+ return ratisTransferLeaderTimeoutMs;
+ }
+
+ public void setRatisTransferLeaderTimeoutMs(int
ratisTransferLeaderTimeoutMs) {
+ this.ratisTransferLeaderTimeoutMs = ratisTransferLeaderTimeoutMs;
+ }
+
public long getConfigNodeRatisLogMax() {
return configNodeRatisLogMax;
}
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
index f22067c1bf4..0c9ccdeb928 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
@@ -652,6 +652,12 @@ public class ConfigNodeDescriptor {
"ratis_first_election_timeout_max_ms",
String.valueOf(conf.getRatisFirstElectionTimeoutMaxMs()))));
+ conf.setRatisTransferLeaderTimeoutMs(
+ Integer.parseInt(
+ properties.getProperty(
+ "ratis_transfer_leader_timeout_ms",
+ String.valueOf(conf.getRatisTransferLeaderTimeoutMs()))));
+
conf.setConfigNodeRatisLogMax(
Long.parseLong(
properties.getProperty(
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
index 0189e33ceb6..d87a9316523 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
@@ -141,6 +141,11 @@ public class ConsensusManager {
.setConsensusGroupType(TConsensusGroupType.ConfigRegion)
.setRatisConfig(
RatisConfig.newBuilder()
+ .setUtils(
+ RatisConfig.Utils.newBuilder()
+ .setTransferLeaderTimeoutMs(
+
CONF.getRatisTransferLeaderTimeoutMs())
+ .build())
.setLeaderLogAppender(
RatisConfig.LeaderLogAppender.newBuilder()
.setBufferByteLimit(
diff --git
a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/RatisConfig.java
b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/RatisConfig.java
index 3bb4e64e490..2161243fda9 100644
---
a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/RatisConfig.java
+++
b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/RatisConfig.java
@@ -196,8 +196,9 @@ public class RatisConfig {
return this;
}
- public void setUtils(Utils utils) {
+ public Builder setUtils(Utils utils) {
this.utils = utils;
+ return this;
}
}
@@ -1096,10 +1097,13 @@ public class RatisConfig {
private final int sleepDeviationThresholdMs;
private final int closeThresholdMs;
+ private final int transferLeaderTimeoutMs;
- private Utils(int sleepDeviationThresholdMs, int closeThresholdMs) {
+ private Utils(
+ int sleepDeviationThresholdMs, int closeThresholdMs, int
transferLeaderTimeoutMs) {
this.sleepDeviationThresholdMs = sleepDeviationThresholdMs;
this.closeThresholdMs = closeThresholdMs;
+ this.transferLeaderTimeoutMs = transferLeaderTimeoutMs;
}
public int getSleepDeviationThresholdMs() {
@@ -1110,6 +1114,10 @@ public class RatisConfig {
return closeThresholdMs;
}
+ public int getTransferLeaderTimeoutMs() {
+ return transferLeaderTimeoutMs;
+ }
+
public static Utils.Builder newBuilder() {
return new Utils.Builder();
}
@@ -1119,16 +1127,25 @@ public class RatisConfig {
private int sleepDeviationThresholdMs = 4 * 1000;
private int closeThresholdMs = Integer.MAX_VALUE;
+ private int transferLeaderTimeoutMs = 30 * 1000;
+
public Utils build() {
- return new Utils(sleepDeviationThresholdMs, closeThresholdMs);
+ return new Utils(sleepDeviationThresholdMs, closeThresholdMs,
transferLeaderTimeoutMs);
}
- public void setSleepDeviationThresholdMs(int sleepDeviationThresholdMs) {
+ public Utils.Builder setSleepDeviationThresholdMs(int
sleepDeviationThresholdMs) {
this.sleepDeviationThresholdMs = sleepDeviationThresholdMs;
+ return this;
}
- public void setCloseThresholdMs(int closeThresholdMs) {
+ public Utils.Builder setCloseThresholdMs(int closeThresholdMs) {
this.closeThresholdMs = closeThresholdMs;
+ return this;
+ }
+
+ public Utils.Builder setTransferLeaderTimeoutMs(int
transferLeaderTimeoutMs) {
+ this.transferLeaderTimeoutMs = transferLeaderTimeoutMs;
+ return this;
}
}
}
diff --git
a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/exception/RatisRequestFailedException.java
b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/exception/RatisRequestFailedException.java
index 1064f76e72f..46d47a4b08b 100644
---
a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/exception/RatisRequestFailedException.java
+++
b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/exception/RatisRequestFailedException.java
@@ -29,4 +29,13 @@ public class RatisRequestFailedException extends
ConsensusException {
+
Optional.ofNullable(cause).map(Exception::getMessage).orElse("Unknown"),
cause);
}
+
+ public RatisRequestFailedException(String message, Exception cause) {
+ super(
+ "Ratis request failed: "
+ + message
+ + ". "
+ +
Optional.ofNullable(cause).map(Exception::getMessage).orElse("Unknown"),
+ cause);
+ }
}
diff --git
a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java
b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java
index ff16434df14..62d382babb9 100644
---
a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java
+++
b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java
@@ -141,7 +141,7 @@ class RatisConsensus implements IConsensus {
private final RatisConfig.Read.Option readOption;
private final RetryPolicy<RaftClientReply> readRetryPolicy;
private final RetryPolicy<RaftClientReply> writeRetryPolicy;
-
+ private final int transferLeadershipTimeoutMs;
private final RatisMetricSet ratisMetricSet;
private final TConsensusGroupType consensusGroupType;
@@ -165,6 +165,8 @@ class RatisConsensus implements IConsensus {
this.readOption == RatisConfig.Read.Option.DEFAULT ? new
ConcurrentHashMap<>() : null;
this.consensusGroupType = config.getConsensusGroupType();
this.ratisMetricSet = new RatisMetricSet();
+ this.transferLeadershipTimeoutMs =
+ config.getRatisConfig().getUtils().getTransferLeaderTimeoutMs();
this.readRetryPolicy =
RetryPolicy.<RaftClientReply>newBuilder()
.setRetryHandler(
@@ -704,10 +706,22 @@ class RatisConsensus implements IConsensus {
try {
reply = transferLeader(raftGroup, newRaftLeader);
if (!reply.isSuccess()) {
- throw new RatisRequestFailedException(reply.getException());
+ String errorMsg =
+ String.format(
+ "transferLeader for group %s to %s failed. This could be due
to a timeout, "
+ + "especially during heavy disk usage. Consider increasing
the "
+ + "'ratis_transfer_leader_timeout_ms' configuration
property.",
+ groupId, newLeader);
+ throw new RatisRequestFailedException(errorMsg, reply.getException());
}
} catch (Exception e) {
- throw new RatisRequestFailedException(e);
+ String errorMsg =
+ String.format(
+ "transferLeader for group %s to %s failed. This could be due to
a timeout, "
+ + "especially during initial startup. Consider increasing
the "
+ + "'ratis_rpc_transfer_leader_timeout_ms' configuration
property.",
+ groupId, newLeader);
+ throw new RatisRequestFailedException(errorMsg, e);
}
}
@@ -722,7 +736,8 @@ class RatisConsensus implements IConsensus {
return client
.getRaftClient()
.admin()
- .transferLeadership(newLeader != null ? newLeader.getId() : null,
10000);
+ .transferLeadership(
+ newLeader != null ? newLeader.getId() : null,
transferLeadershipTimeoutMs);
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 3bfa24641b9..a8a4c796557 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -1039,6 +1039,8 @@ public class IoTDBConfig {
private long dataRatisPeriodicSnapshotInterval = 24L * 60 * 60; // 24hr
private long schemaRatisPeriodicSnapshotInterval = 24L * 60 * 60; // 24hr
+ private int ratisTransferLeaderTimeoutMs = 30 * 1000; // 30s
+
/** whether to enable the audit log * */
private boolean enableAuditLog = false;
@@ -4077,6 +4079,14 @@ public class IoTDBConfig {
this.schemaRatisPeriodicSnapshotInterval =
schemaRatisPeriodicSnapshotInterval;
}
+ public int getRatisTransferLeaderTimeoutMs() {
+ return ratisTransferLeaderTimeoutMs;
+ }
+
+ public void setRatisTransferLeaderTimeoutMs(int
ratisTransferLeaderTimeoutMs) {
+ this.ratisTransferLeaderTimeoutMs = ratisTransferLeaderTimeoutMs;
+ }
+
public boolean isEnableTsFileValidation() {
return enableTsFileValidation;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index f9377892ef3..17e01d1e340 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -2684,6 +2684,8 @@ public class IoTDBDescriptor {
conf.setSchemaRatisPeriodicSnapshotInterval(
ratisConfig.getSchemaRegionPeriodicSnapshotInterval());
conf.setDataRatisPeriodicSnapshotInterval(ratisConfig.getDataRegionPeriodicSnapshotInterval());
+
+
conf.setRatisTransferLeaderTimeoutMs(ratisConfig.getRatisTransferLeaderTimeoutMs());
}
public void loadCQConfig(TCQConfig cqConfig) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
index 4785f03c483..7b3ddc372fa 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
@@ -184,6 +184,10 @@ public class DataRegionConsensusImpl {
// An empty log is committed after each restart, even if no
data is
// written. This setting ensures that compaction work is not
discarded
// even if there are frequent restarts
+ .setUtils(
+ RatisConfig.Utils.newBuilder()
+
.setTransferLeaderTimeoutMs(CONF.getRatisTransferLeaderTimeoutMs())
+ .build())
.setSnapshot(
Snapshot.newBuilder()
.setCreationGap(1)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/SchemaRegionConsensusImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/SchemaRegionConsensusImpl.java
index b3e79fa1eec..371329d4738 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/SchemaRegionConsensusImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/SchemaRegionConsensusImpl.java
@@ -77,6 +77,11 @@ public class SchemaRegionConsensusImpl {
.setConsensusGroupType(TConsensusGroupType.SchemaRegion)
.setRatisConfig(
RatisConfig.newBuilder()
+ .setUtils(
+ RatisConfig.Utils.newBuilder()
+ .setTransferLeaderTimeoutMs(
+
CONF.getRatisTransferLeaderTimeoutMs())
+ .build())
.setSnapshot(
RatisConfig.Snapshot.newBuilder()
.setAutoTriggerThreshold(
diff --git
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index f0957132d2d..e114686e20c 100644
---
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1946,6 +1946,10 @@ data_region_ratis_max_sleep_time_ms=10000
ratis_first_election_timeout_min_ms=50
ratis_first_election_timeout_max_ms=150
+# the timeout threshold for transferring leader in ratis
+# Datatype: int
+ratis_transfer_leader_timeout_ms=30000
+
# preserve certain logs when take snapshot and purge
# effectiveMode: restart
# Datatype: int
diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
index b869d29cde3..2a5682926df 100644
--- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
@@ -93,6 +93,8 @@ struct TRatisConfig {
33: required i64 schemaRegionPeriodicSnapshotInterval
34: required i64 dataRegionPeriodicSnapshotInterval
+
+ 35: required i32 ratisTransferLeaderTimeoutMs;
}
struct TCQConfig {
