This is an automated email from the ASF dual-hosted git repository.
cmccabe pushed a change to branch 3.4
in repository https://gitbox.apache.org/repos/asf/kafka.git
from 385c317df3e KAFKA-14496: Wrong Base64 encoder used by OIDC
OAuthBearerLoginCallbackHandler (#13000)
new e94a25173dc MINOR: Move dynamic config logic to DynamicConfigPublisher
(#12958)
new 4b03c8c4c3d KAFKA-14427 ZK client support for migrations (#12946)
new e49132326e1 MINOR: Move MetadataQuorumCommand from `core` to `tools`
(#12951)
new 984c5eda7c0 KAFKA-14392: Fix overly long request timeouts in
BrokerToControllerChannelManager (#12856)
new 17e6bba20b4 MINOR: Introduce MetadataProvenance and ImageReWriter
(#12964)
new 793c36c5548 MINOR; Improve high watermark log messages (#12975)
new 2de4451e27e MINOR: Pass snapshot ID directly in
`RaftClient.createSnapshot` (#12981)
new 1d528e395a9 KAFKA-14448 Let ZK brokers register with KRaft controller
(#12965)
new 47c5b001be1 MINOR: Change KRaft ZK controller registration algorithm
(#12973)
new aa532b44199 KAFKA-14446: API forwarding support from zkBrokers to the
Controller (#12961)
new 59bad9e47f2 MINOR: ControllerServer should use the new metadata loader
and snapshot generator (#12983)
The 11 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
bin/kafka-metadata-quorum.sh | 2 +-
bin/windows/kafka-metatada-quorum.bat | 2 +-
build.gradle | 7 +-
checkstyle/import-control.xml | 10 +-
.../common/requests/AbstractControlRequest.java | 10 +-
.../common/requests/BrokerRegistrationRequest.java | 9 +
.../kafka/common/requests/LeaderAndIsrRequest.java | 18 +-
.../kafka/common/requests/StopReplicaRequest.java | 18 +-
.../common/requests/UpdateMetadataRequest.java | 26 +-
.../java/org/apache/kafka/common/utils/Utils.java | 14 +-
.../common/message/BrokerRegistrationRequest.json | 4 +-
.../common/message/BrokerRegistrationResponse.json | 2 +-
.../resources/common/message/EnvelopeRequest.json | 2 +-
.../common/message/LeaderAndIsrRequest.json | 4 +-
.../common/message/StopReplicaRequest.json | 4 +-
.../common/message/UpdateMetadataRequest.json | 4 +-
.../common/requests/ApiVersionsResponseTest.java | 1 -
.../org/apache/kafka/common/utils/UtilsTest.java | 7 +
core/src/main/scala/kafka/Kafka.scala | 8 +-
.../scala/kafka/admin/MetadataQuorumCommand.scala | 172 -------
.../scala/kafka/common/InterBrokerSendThread.scala | 2 +-
.../kafka/server/BrokerLifecycleManager.scala | 20 +-
.../src/main/scala/kafka/server/BrokerServer.scala | 30 +-
.../server/BrokerToControllerChannelManager.scala | 123 +++--
.../main/scala/kafka/server/ControllerApis.scala | 2 +-
.../main/scala/kafka/server/ControllerServer.scala | 12 +-
core/src/main/scala/kafka/server/KafkaApis.scala | 18 +-
core/src/main/scala/kafka/server/KafkaConfig.scala | 11 +-
core/src/main/scala/kafka/server/KafkaServer.scala | 133 ++++-
.../main/scala/kafka/server/MetadataCache.scala | 18 +-
.../src/main/scala/kafka/server/SharedServer.scala | 99 +++-
.../main/scala/kafka/server/ZkAdminManager.scala | 27 +-
.../server/metadata/BrokerMetadataListener.scala | 56 +-
.../server/metadata/BrokerMetadataPublisher.scala | 61 +--
.../server/metadata/BrokerServerMetrics.scala | 59 ++-
.../server/metadata/DynamicConfigPublisher.scala | 103 ++++
.../kafka/server/metadata/KRaftMetadataCache.scala | 11 +-
.../server/metadata/OffsetTrackingListener.scala | 48 ++
.../kafka/server/metadata/ZkMetadataCache.scala | 58 ++-
core/src/main/scala/kafka/zk/KafkaZkClient.scala | 376 ++++++++++++--
core/src/main/scala/kafka/zk/ZkData.scala | 24 +-
.../main/scala/kafka/zk/ZkMigrationClient.scala | 438 ++++++++++++++++
core/src/test/java/kafka/test/ClusterInstance.java | 2 +
core/src/test/java/kafka/test/MockController.java | 5 -
.../test/junit/RaftClusterInvocationContext.java | 8 +
.../test/junit/ZkClusterInvocationContext.java | 8 +-
.../kafka/api/IntegrationTestHarness.scala | 4 +
.../kafka/api/PlaintextAdminIntegrationTest.scala | 2 +-
.../server/KafkaServerKRaftRegistrationTest.scala | 85 +++
.../kafka/server/QuorumTestHarness.scala | 4 +
.../kafka/zk/ZkMigrationIntegrationTest.scala | 105 ++++
.../BrokerToControllerRequestThreadTest.scala | 52 +-
.../server/metadata/BrokerServerMetricsTest.scala | 21 +-
.../src/test/scala/kafka/utils/TestInfoUtils.scala | 8 +
.../kafka/admin/MetadataQuorumCommandTest.scala | 192 -------
.../kafka/integration/KafkaServerTestHarness.scala | 3 +-
.../unit/kafka/server/ApiVersionManagerTest.scala | 6 +-
.../kafka/server/BrokerLifecycleManagerTest.scala | 9 +-
.../server/BrokerRegistrationRequestTest.scala | 202 ++++++++
.../kafka/server/CreateTopicsRequestTest.scala | 10 +-
.../kafka/server/DeleteTopicsRequestTest.scala | 5 +-
.../kafka/server/DynamicConfigChangeTest.scala | 4 +-
.../unit/kafka/server/ForwardingManagerTest.scala | 24 +-
.../scala/unit/kafka/server/KafkaApisTest.scala | 2 +-
.../unit/kafka/server/MetadataCacheTest.scala | 25 +-
.../MockBrokerToControllerChannelManager.scala | 2 +-
.../server/ReplicaManagerConcurrencyTest.scala | 8 +-
.../unit/kafka/server/ReplicaManagerTest.scala | 5 +-
.../scala/unit/kafka/server/RequestQuotaTest.scala | 8 +-
.../unit/kafka/server/ZkAdminManagerTest.scala | 28 +-
.../metadata/BrokerMetadataListenerTest.scala | 8 +-
.../metadata/BrokerMetadataPublisherTest.scala | 45 +-
.../metadata/BrokerMetadataSnapshotterTest.scala | 18 +-
.../test/scala/unit/kafka/utils/TestUtils.scala | 7 +-
.../scala/unit/kafka/zk/KafkaZkClientTest.scala | 119 ++++-
.../unit/kafka/zk/ZkMigrationClientTest.scala | 362 +++++++++++++
.../jmh/fetcher/ReplicaFetcherThreadBenchmark.java | 3 +-
.../jmh/metadata/MetadataRequestBenchmark.java | 3 +-
.../apache/kafka/jmh/server/CheckpointBench.java | 4 +-
.../kafka/jmh/server/PartitionCreationBench.java | 6 +-
.../apache/kafka/controller/AclControlManager.java | 18 -
.../controller/ClientQuotaControlManager.java | 42 --
.../kafka/controller/ClusterControlManager.java | 87 +---
.../controller/ConfigurationControlManager.java | 36 --
.../org/apache/kafka/controller/Controller.java | 8 -
.../kafka/controller/FeatureControlManager.java | 44 --
.../apache/kafka/controller/LogReplayTracker.java | 3 +-
.../kafka/controller/ProducerIdControlManager.java | 24 +-
.../apache/kafka/controller/QuorumController.java | 315 +-----------
.../controller/ReplicationControlManager.java | 34 --
.../apache/kafka/controller/SnapshotGenerator.java | 132 -----
.../org/apache/kafka/image/ClientQuotasImage.java | 3 +-
.../java/org/apache/kafka/image/MetadataDelta.java | 51 +-
.../java/org/apache/kafka/image/MetadataImage.java | 26 +-
.../org/apache/kafka/image/MetadataProvenance.java | 92 ++++
.../apache/kafka/image/MetadataVersionChange.java | 77 +++
.../image/MetadataVersionChangeException.java | 22 +-
.../kafka/image/loader/LogDeltaManifest.java | 106 ++++
.../apache/kafka/image/loader/MetadataLoader.java | 572 +++++++++++++++++++++
.../kafka/image/loader/MetadataLoaderMetrics.java | 34 +-
.../kafka/image/loader/SnapshotManifest.java | 77 +++
.../kafka/image/publisher/MetadataPublisher.java | 72 +++
.../kafka/image/publisher/SnapshotEmitter.java | 123 +++++
.../kafka/image/publisher/SnapshotGenerator.java | 275 ++++++++++
.../{RecordListWriter.java => ImageReWriter.java} | 34 +-
.../kafka/image/writer/ImageWriterOptions.java | 17 +-
.../apache/kafka/metadata/BrokerRegistration.java | 55 +-
.../kafka/metadata/migration/MigrationClient.java | 91 ++++
.../migration/ZkMigrationLeadershipState.java | 49 +-
.../kafka/controller/AclControlManagerTest.java | 24 +-
.../controller/ClientQuotaControlManagerTest.java | 59 ++-
.../controller/ClusterControlManagerTest.java | 27 +-
.../ConfigurationControlManagerTest.java | 8 -
.../controller/FeatureControlManagerTest.java | 38 +-
.../controller/ProducerIdControlManagerTest.java | 30 +-
.../kafka/controller/QuorumControllerTest.java | 401 ++-------------
.../controller/ReplicationControlManagerTest.java | 11 -
.../kafka/controller/SnapshotGeneratorTest.java | 112 ----
.../org/apache/kafka/image/ImageDowngradeTest.java | 4 +-
.../org/apache/kafka/image/MetadataImageTest.java | 34 +-
.../kafka/image/MetadataVersionChangeTest.java | 63 +++
.../kafka/image/loader/MetadataLoaderTest.java | 517 +++++++++++++++++++
.../kafka/image/publisher/SnapshotEmitterTest.java | 206 ++++++++
.../image/publisher/SnapshotGeneratorTest.java | 185 +++++++
...dListWriterTest.java => ImageReWriterTest.java} | 22 +-
.../kafka/image/writer/ImageWriterOptionsTest.java | 52 ++
.../kafka/metadata/BrokerRegistrationTest.java | 4 +-
.../org/apache/kafka/metadata/RecordTestUtils.java | 48 --
.../org/apache/kafka/metalog/LocalLogManager.java | 27 +-
.../kafka/metalog/LocalLogManagerTestEnv.java | 7 +
.../java/org/apache/kafka/raft/FollowerState.java | 69 ++-
.../org/apache/kafka/raft/KafkaRaftClient.java | 5 +-
.../java/org/apache/kafka/raft/LeaderState.java | 28 +-
.../java/org/apache/kafka/raft/RaftClient.java | 23 +-
.../org/apache/kafka/raft/ReplicatedCounter.java | 3 +-
.../kafka/snapshot/RecordsSnapshotWriter.java | 46 +-
.../org/apache/kafka/snapshot/SnapshotWriter.java | 2 +-
.../kafka/raft/KafkaRaftClientSnapshotTest.java | 48 +-
.../kafka/snapshot/SnapshotWriterReaderTest.java | 8 +-
.../kafka/server/common/MetadataVersion.java | 13 +-
.../org/apache/kafka/server/util/ToolsUtils.java | 19 +-
.../kafka/server/common/MetadataVersionTest.java | 10 +-
tests/kafkatest/tests/core/snapshot_test.py | 2 +-
.../apache/kafka/tools/MetadataQuorumCommand.java | 195 +++++++
.../org/apache/kafka/tools/TerseException.java | 15 +-
.../apache/kafka/tools/TransactionsCommand.java | 42 +-
.../tools/MetadataQuorumCommandErrorTest.java | 48 ++
.../kafka/tools/MetadataQuorumCommandTest.java | 161 ++++++
.../org/apache/kafka/tools/ToolsTestUtils.java | 51 ++
.../kafka/tools/TransactionsCommandTest.java | 12 +-
150 files changed, 6250 insertions(+), 2413 deletions(-)
delete mode 100644 core/src/main/scala/kafka/admin/MetadataQuorumCommand.scala
create mode 100644
core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala
create mode 100644
core/src/main/scala/kafka/server/metadata/OffsetTrackingListener.scala
create mode 100644 core/src/main/scala/kafka/zk/ZkMigrationClient.scala
create mode 100644
core/src/test/scala/integration/kafka/server/KafkaServerKRaftRegistrationTest.scala
create mode 100644
core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala
delete mode 100644
core/src/test/scala/unit/kafka/admin/MetadataQuorumCommandTest.scala
create mode 100644
core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala
create mode 100644
core/src/test/scala/unit/kafka/zk/ZkMigrationClientTest.scala
delete mode 100644
metadata/src/main/java/org/apache/kafka/controller/SnapshotGenerator.java
create mode 100644
metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java
create mode 100644
metadata/src/main/java/org/apache/kafka/image/MetadataVersionChange.java
copy
streams/src/main/java/org/apache/kafka/streams/errors/UnknownStateStoreException.java
=>
metadata/src/main/java/org/apache/kafka/image/MetadataVersionChangeException.java
(64%)
create mode 100644
metadata/src/main/java/org/apache/kafka/image/loader/LogDeltaManifest.java
create mode 100644
metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java
copy generator/src/main/java/org/apache/kafka/message/MessageSpecType.java =>
metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoaderMetrics.java
(57%)
create mode 100644
metadata/src/main/java/org/apache/kafka/image/loader/SnapshotManifest.java
create mode 100644
metadata/src/main/java/org/apache/kafka/image/publisher/MetadataPublisher.java
create mode 100644
metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java
create mode 100644
metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java
copy
metadata/src/main/java/org/apache/kafka/image/writer/{RecordListWriter.java =>
ImageReWriter.java} (61%)
create mode 100644
metadata/src/main/java/org/apache/kafka/metadata/migration/MigrationClient.java
delete mode 100644
metadata/src/test/java/org/apache/kafka/controller/SnapshotGeneratorTest.java
create mode 100644
metadata/src/test/java/org/apache/kafka/image/MetadataVersionChangeTest.java
create mode 100644
metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java
create mode 100644
metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java
create mode 100644
metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java
copy
metadata/src/test/java/org/apache/kafka/image/writer/{RecordListWriterTest.java
=> ImageReWriterTest.java} (71%)
create mode 100644
metadata/src/test/java/org/apache/kafka/image/writer/ImageWriterOptionsTest.java
create mode 100644
tools/src/main/java/org/apache/kafka/tools/MetadataQuorumCommand.java
copy
clients/src/main/java/org/apache/kafka/common/errors/GroupMaxSizeReachedException.java
=> tools/src/main/java/org/apache/kafka/tools/TerseException.java (67%)
create mode 100644
tools/src/test/java/org/apache/kafka/tools/MetadataQuorumCommandErrorTest.java
create mode 100644
tools/src/test/java/org/apache/kafka/tools/MetadataQuorumCommandTest.java
create mode 100644
tools/src/test/java/org/apache/kafka/tools/ToolsTestUtils.java
