This is an automated email from the ASF dual-hosted git repository.
chia7712 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2a0dbd8e0b0 KAFKA-18909 Move DynamicThreadPool to server module
(#19081)
2a0dbd8e0b0 is described below
commit 2a0dbd8e0b0fbc4f4c7b9b771c15362744cfa4e8
Author: ClarkChen <[email protected]>
AuthorDate: Sun Mar 9 17:42:51 2025 +0800
KAFKA-18909 Move DynamicThreadPool to server module (#19081)
* Add `DynamicThreadPool.java` to the server module.
* Remove the old DynamicThreadPool object in the
`DynamicBrokerConfig.scala`.
Reviewers: TengYao Chi <[email protected]>, Chia-Ping Tsai
<[email protected]>
---
.../scala/kafka/server/DynamicBrokerConfig.scala | 44 ++------------
core/src/main/scala/kafka/server/KafkaConfig.scala | 4 --
.../kafka/server/DynamicBrokerConfigTest.scala | 3 +-
.../org/apache/kafka/server/DynamicThreadPool.java | 68 ++++++++++++++++++++++
.../kafka/server/config/AbstractKafkaConfig.java | 16 +++++
5 files changed, 90 insertions(+), 45 deletions(-)
diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
index 30d37a0b6d1..33b7192c36d 100755
--- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
+++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala
@@ -35,8 +35,8 @@ import
org.apache.kafka.common.security.authenticator.LoginManager
import org.apache.kafka.common.utils.{ConfigUtils, Utils}
import org.apache.kafka.coordinator.transaction.TransactionLogConfig
import org.apache.kafka.network.SocketServerConfigs
-import org.apache.kafka.server.ProcessRole
-import org.apache.kafka.server.config.{ReplicationConfigs, ServerConfigs,
ServerLogConfigs, ServerTopicConfigSynonyms}
+import org.apache.kafka.server.{ProcessRole, DynamicThreadPool}
+import org.apache.kafka.server.config.{ServerConfigs, ServerLogConfigs,
ServerTopicConfigSynonyms}
import org.apache.kafka.server.log.remote.storage.RemoteLogManagerConfig
import org.apache.kafka.server.metrics.{ClientMetricsReceiverPlugin,
MetricConfigs}
import org.apache.kafka.server.telemetry.ClientTelemetry
@@ -85,7 +85,7 @@ object DynamicBrokerConfig {
val AllDynamicConfigs = DynamicSecurityConfigs ++
LogCleaner.ReconfigurableConfigs ++
DynamicLogConfig.ReconfigurableConfigs ++
- DynamicThreadPool.ReconfigurableConfigs ++
+ DynamicThreadPool.RECONFIGURABLE_CONFIGS.asScala ++
Set(MetricConfigs.METRIC_REPORTER_CLASSES_CONFIG) ++
DynamicListenerConfig.ReconfigurableConfigs ++
SocketServer.ReconfigurableConfigs ++
@@ -641,42 +641,6 @@ class DynamicLogConfig(logManager: LogManager) extends
BrokerReconfigurable with
}
}
-object DynamicThreadPool {
- val ReconfigurableConfigs = Set(
- ServerConfigs.NUM_IO_THREADS_CONFIG,
- ReplicationConfigs.NUM_REPLICA_FETCHERS_CONFIG,
- ServerLogConfigs.NUM_RECOVERY_THREADS_PER_DATA_DIR_CONFIG,
- ServerConfigs.BACKGROUND_THREADS_CONFIG)
-
- def validateReconfiguration(currentConfig: KafkaConfig, newConfig:
KafkaConfig): Unit = {
- newConfig.values.forEach { (k, v) =>
- if (ReconfigurableConfigs.contains(k)) {
- val newValue = v.asInstanceOf[Int]
- val oldValue = getValue(currentConfig, k)
- if (newValue != oldValue) {
- val errorMsg = s"Dynamic thread count update validation failed for
$k=$v"
- if (newValue <= 0)
- throw new ConfigException(s"$errorMsg, value should be at least 1")
- if (newValue < oldValue / 2)
- throw new ConfigException(s"$errorMsg, value should be at least
half the current value $oldValue")
- if (newValue > oldValue * 2)
- throw new ConfigException(s"$errorMsg, value should not be greater
than double the current value $oldValue")
- }
- }
- }
- }
-
- def getValue(config: KafkaConfig, name: String): Int = {
- name match {
- case ServerConfigs.NUM_IO_THREADS_CONFIG => config.numIoThreads
- case ReplicationConfigs.NUM_REPLICA_FETCHERS_CONFIG =>
config.numReplicaFetchers
- case ServerLogConfigs.NUM_RECOVERY_THREADS_PER_DATA_DIR_CONFIG =>
config.numRecoveryThreadsPerDataDir
- case ServerConfigs.BACKGROUND_THREADS_CONFIG => config.backgroundThreads
- case n => throw new IllegalStateException(s"Unexpected config $n")
- }
- }
-}
-
class ControllerDynamicThreadPool(controller: ControllerServer) extends
BrokerReconfigurable {
override def reconfigurableConfigs: Set[String] = {
@@ -696,7 +660,7 @@ class ControllerDynamicThreadPool(controller:
ControllerServer) extends BrokerRe
class BrokerDynamicThreadPool(server: KafkaBroker) extends
BrokerReconfigurable {
override def reconfigurableConfigs: Set[String] = {
- DynamicThreadPool.ReconfigurableConfigs
+ DynamicThreadPool.RECONFIGURABLE_CONFIGS.asScala
}
override def validateReconfiguration(newConfig: KafkaConfig): Unit = {
diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala
b/core/src/main/scala/kafka/server/KafkaConfig.scala
index 9e45dfcc3fa..6d120d5b609 100755
--- a/core/src/main/scala/kafka/server/KafkaConfig.scala
+++ b/core/src/main/scala/kafka/server/KafkaConfig.scala
@@ -259,8 +259,6 @@ class KafkaConfig private(doLog: Boolean, val props:
util.Map[_, _])
def metadataLogSegmentMinBytes =
getInt(KRaftConfigs.METADATA_LOG_SEGMENT_MIN_BYTES_CONFIG)
val serverMaxStartupTimeMs =
getLong(KRaftConfigs.SERVER_MAX_STARTUP_TIME_MS_CONFIG)
- def backgroundThreads = getInt(ServerConfigs.BACKGROUND_THREADS_CONFIG)
- def numIoThreads = getInt(ServerConfigs.NUM_IO_THREADS_CONFIG)
def messageMaxBytes = getInt(ServerConfigs.MESSAGE_MAX_BYTES_CONFIG)
val requestTimeoutMs = getInt(ServerConfigs.REQUEST_TIMEOUT_MS_CONFIG)
val connectionSetupTimeoutMs =
getLong(ServerConfigs.SOCKET_CONNECTION_SETUP_TIMEOUT_MS_CONFIG)
@@ -333,7 +331,6 @@ class KafkaConfig private(doLog: Boolean, val props:
util.Map[_, _])
def logSegmentBytes = getInt(ServerLogConfigs.LOG_SEGMENT_BYTES_CONFIG)
def logFlushIntervalMessages =
getLong(ServerLogConfigs.LOG_FLUSH_INTERVAL_MESSAGES_CONFIG)
def logCleanerThreads = getInt(CleanerConfig.LOG_CLEANER_THREADS_PROP)
- def numRecoveryThreadsPerDataDir =
getInt(ServerLogConfigs.NUM_RECOVERY_THREADS_PER_DATA_DIR_CONFIG)
val logFlushSchedulerIntervalMs =
getLong(ServerLogConfigs.LOG_FLUSH_SCHEDULER_INTERVAL_MS_CONFIG)
val logFlushOffsetCheckpointIntervalMs =
getInt(ServerLogConfigs.LOG_FLUSH_OFFSET_CHECKPOINT_INTERVAL_MS_CONFIG).toLong
val logFlushStartOffsetCheckpointIntervalMs =
getInt(ServerLogConfigs.LOG_FLUSH_START_OFFSET_CHECKPOINT_INTERVAL_MS_CONFIG).toLong
@@ -380,7 +377,6 @@ class KafkaConfig private(doLog: Boolean, val props:
util.Map[_, _])
val replicaFetchMinBytes =
getInt(ReplicationConfigs.REPLICA_FETCH_MIN_BYTES_CONFIG)
val replicaFetchResponseMaxBytes =
getInt(ReplicationConfigs.REPLICA_FETCH_RESPONSE_MAX_BYTES_CONFIG)
val replicaFetchBackoffMs =
getInt(ReplicationConfigs.REPLICA_FETCH_BACKOFF_MS_CONFIG)
- def numReplicaFetchers =
getInt(ReplicationConfigs.NUM_REPLICA_FETCHERS_CONFIG)
val replicaHighWatermarkCheckpointIntervalMs =
getLong(ReplicationConfigs.REPLICA_HIGH_WATERMARK_CHECKPOINT_INTERVAL_MS_CONFIG)
val fetchPurgatoryPurgeIntervalRequests =
getInt(ReplicationConfigs.FETCH_PURGATORY_PURGE_INTERVAL_REQUESTS_CONFIG)
val producerPurgatoryPurgeIntervalRequests =
getInt(ReplicationConfigs.PRODUCER_PURGATORY_PURGE_INTERVAL_REQUESTS_CONFIG)
diff --git
a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
index 7abe369b6d9..816752b2706 100755
--- a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala
@@ -33,6 +33,7 @@ import org.apache.kafka.common.network.ListenerName
import org.apache.kafka.common.security.auth.SecurityProtocol
import org.apache.kafka.raft.QuorumConfig
import org.apache.kafka.network.SocketServerConfigs
+import org.apache.kafka.server.DynamicThreadPool
import org.apache.kafka.server.authorizer._
import org.apache.kafka.server.config.{KRaftConfigs, ReplicationConfigs,
ServerConfigs, ServerLogConfigs}
import org.apache.kafka.server.log.remote.storage.RemoteLogManagerConfig
@@ -1041,7 +1042,7 @@ class DynamicBrokerConfigTest {
class TestDynamicThreadPool extends BrokerReconfigurable {
override def reconfigurableConfigs: Set[String] = {
- DynamicThreadPool.ReconfigurableConfigs
+ DynamicThreadPool.RECONFIGURABLE_CONFIGS.asScala
}
override def reconfigure(oldConfig: KafkaConfig, newConfig: KafkaConfig):
Unit = {
diff --git
a/server/src/main/java/org/apache/kafka/server/DynamicThreadPool.java
b/server/src/main/java/org/apache/kafka/server/DynamicThreadPool.java
new file mode 100644
index 00000000000..c267d8762dc
--- /dev/null
+++ b/server/src/main/java/org/apache/kafka/server/DynamicThreadPool.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server;
+
+import org.apache.kafka.common.config.ConfigException;
+import org.apache.kafka.server.config.AbstractKafkaConfig;
+import org.apache.kafka.server.config.ReplicationConfigs;
+import org.apache.kafka.server.config.ServerConfigs;
+import org.apache.kafka.server.config.ServerLogConfigs;
+
+import java.util.Map;
+import java.util.Set;
+
+public class DynamicThreadPool {
+ public static final Set<String> RECONFIGURABLE_CONFIGS = Set.of(
+ ServerConfigs.NUM_IO_THREADS_CONFIG,
+ ReplicationConfigs.NUM_REPLICA_FETCHERS_CONFIG,
+ ServerLogConfigs.NUM_RECOVERY_THREADS_PER_DATA_DIR_CONFIG,
+ ServerConfigs.BACKGROUND_THREADS_CONFIG
+ );
+
+ private DynamicThreadPool() {}
+
+ public static void validateReconfiguration(AbstractKafkaConfig
currentConfig, AbstractKafkaConfig newConfig) {
+ for (Map.Entry<String, ?> entry : newConfig.values().entrySet()) {
+ String key = entry.getKey();
+ if (RECONFIGURABLE_CONFIGS.contains(key)) {
+ int newValue = (int) entry.getValue();
+ int oldValue = getValue(currentConfig, key);
+
+ if (newValue != oldValue) {
+ String errorMsg = String.format("Dynamic thread count
update validation failed for %s=%d", key, newValue);
+
+ if (newValue <= 0)
+ throw new ConfigException(String.format("%s, value
should be at least 1", errorMsg));
+ if (newValue < oldValue / 2)
+ throw new ConfigException(String.format("%s, value
should be at least half the current value %d", errorMsg, oldValue));
+ if (newValue > oldValue * 2)
+ throw new ConfigException(String.format("%s, value
should not be greater than double the current value %d", errorMsg, oldValue));
+ }
+ }
+ }
+ }
+
+ public static int getValue(AbstractKafkaConfig config, String name) {
+ return switch (name) {
+ case ServerConfigs.NUM_IO_THREADS_CONFIG -> config.numIoThreads();
+ case ReplicationConfigs.NUM_REPLICA_FETCHERS_CONFIG ->
config.numReplicaFetchers();
+ case ServerLogConfigs.NUM_RECOVERY_THREADS_PER_DATA_DIR_CONFIG ->
config.numRecoveryThreadsPerDataDir();
+ case ServerConfigs.BACKGROUND_THREADS_CONFIG ->
config.backgroundThreads();
+ default -> throw new
IllegalStateException(String.format("Unexpected config %s", name));
+ };
+ }
+}
diff --git
a/server/src/main/java/org/apache/kafka/server/config/AbstractKafkaConfig.java
b/server/src/main/java/org/apache/kafka/server/config/AbstractKafkaConfig.java
index 890a62248d5..87bf18a412f 100644
---
a/server/src/main/java/org/apache/kafka/server/config/AbstractKafkaConfig.java
+++
b/server/src/main/java/org/apache/kafka/server/config/AbstractKafkaConfig.java
@@ -67,4 +67,20 @@ public abstract class AbstractKafkaConfig extends
AbstractConfig {
public AbstractKafkaConfig(ConfigDef definition, Map<?, ?> originals,
Map<String, ?> configProviderProps, boolean doLog) {
super(definition, originals, configProviderProps, doLog);
}
+
+ public int numIoThreads() {
+ return getInt(ServerConfigs.NUM_IO_THREADS_CONFIG);
+ }
+
+ public int numReplicaFetchers() {
+ return getInt(ReplicationConfigs.NUM_REPLICA_FETCHERS_CONFIG);
+ }
+
+ public int numRecoveryThreadsPerDataDir() {
+ return
getInt(ServerLogConfigs.NUM_RECOVERY_THREADS_PER_DATA_DIR_CONFIG);
+ }
+
+ public int backgroundThreads() {
+ return getInt(ServerConfigs.BACKGROUND_THREADS_CONFIG);
+ }
}
