This is an automated email from the ASF dual-hosted git repository. cmccabe pushed a commit to branch KAFKA-15183 in repository https://gitbox.apache.org/repos/asf/kafka.git
commit df7b7bde40da7361dd28b0d654211c591381d1a3 Merge: d3aaeae757d 79b8c969e53 Author: Colin P. McCabe <[email protected]> AuthorDate: Mon Jul 24 16:00:16 2023 -0700 Merge branch 'trunk' into KAFKA-15183 LICENSE-binary | 3 +- README.md | 2 +- bin/kafka-delete-records.sh | 2 +- bin/windows/kafka-delete-records.bat | 2 +- build.gradle | 13 +- checkstyle/import-control-core.xml | 1 + checkstyle/import-control-server-common.xml | 3 + checkstyle/import-control.xml | 9 +- checkstyle/suppressions.xml | 14 +- .../consumer/internals/CommitRequestManager.java | 9 +- .../consumer/internals/ConsumerCoordinator.java | 2 +- .../internals/DefaultBackgroundThread.java | 24 +- .../consumer/internals/DefaultEventHandler.java | 3 + .../consumer/internals/PrototypeAsyncConsumer.java | 33 +- .../internals/events/ApplicationEvent.java | 2 +- .../events/ApplicationEventProcessor.java | 30 +- ....java => AssignmentChangeApplicationEvent.java} | 30 +- .../NewTopicsMetadataUpdateRequestEvent.java | 15 +- .../errors/StaleMemberEpochException.java} | 31 +- .../org/apache/kafka/common/protocol/Errors.java | 4 +- .../kafka/common/requests/AbstractRequest.java | 9 +- .../requests/ConsumerGroupHeartbeatRequest.java | 6 +- .../kafka/common/requests/JoinGroupRequest.java | 62 + .../kafka/common/requests/OffsetCommitRequest.java | 49 +- .../kafka/common/utils}/ThroughputThrottler.java | 2 +- .../common/message/OffsetCommitRequest.json | 12 +- .../common/message/OffsetCommitResponse.json | 15 +- .../internals/CommitRequestManagerTest.java | 2 +- .../internals/ConsumerCoordinatorTest.java | 2 +- .../internals/DefaultBackgroundThreadTest.java | 93 +- .../internals/PrototypeAsyncConsumerTest.java | 43 +- .../apache/kafka/common/message/MessageTest.java | 4 +- .../common/requests/OffsetCommitRequestTest.java | 42 +- .../kafka/common/requests/RequestResponseTest.java | 2 +- .../org/apache/kafka/connect/sink/SinkRecord.java | 144 +- .../org/apache/kafka/connect/sink/SinkTask.java | 37 +- .../apache/kafka/connect/sink/SinkTaskContext.java | 2 +- .../apache/kafka/connect/source/SourceTask.java | 7 +- .../kafka/connect/source/SourceTaskContext.java | 2 +- .../apache/kafka/connect/sink/SinkRecordTest.java | 36 +- .../connect/file/FileStreamSinkConnector.java | 8 + .../connect/file/FileStreamSourceConnector.java | 49 + .../file/FileStreamSourceConnectorTest.java | 88 +- .../connect/mirror/MirrorCheckpointConfig.java | 178 +- .../connect/mirror/MirrorConnectorConfig.java | 4 + .../connect/mirror/MirrorHeartbeatConfig.java | 14 +- .../kafka/connect/mirror/MirrorSourceConfig.java | 256 +- .../kafka/connect/runtime/InternalSinkRecord.java | 11 +- .../connect/runtime/isolation/PluginScanner.java | 70 +- .../connect/runtime/isolation/PluginSource.java | 4 +- .../connect/runtime/isolation/PluginUtils.java | 6 +- .../runtime/isolation/ReflectionScanner.java | 85 +- .../runtime/isolation/ServiceLoaderScanner.java | 80 + .../kafka/connect/tools/SchemaSourceTask.java | 2 +- .../kafka/connect/tools/VerifiableSourceTask.java | 2 +- .../integration/MonitorableSourceConnector.java | 2 +- .../connect/runtime/InternalSinkRecordTest.java | 61 + .../kafka/connect/runtime/WorkerSinkTaskTest.java | 43 +- .../runtime/WorkerSinkTaskThreadedTest.java | 2 +- .../runtime/distributed/DistributedConfigTest.java | 11 +- .../runtime/isolation/PluginScannerTest.java | 97 +- .../connect/runtime/isolation/PluginsTest.java | 7 +- .../connect/runtime/isolation/TestPlugins.java | 2 +- .../org.apache.kafka.connect.storage.Converter | 16 + .../org.apache.kafka.connect.storage.Converter | 16 + .../org.apache.kafka.connect.sink.SinkConnector | 21 + .../org.apache.kafka.connect.storage.Converter | 19 + .../org.apache.kafka.connect.storage.Converter | 17 + .../org.apache.kafka.connect.storage.Converter | 16 + .../test/plugins/ReadVersionFromResource.java | 12 +- .../org.apache.kafka.connect.storage.Converter | 16 + .../test/plugins/ReadVersionFromResource.java | 12 +- .../org.apache.kafka.connect.storage.Converter | 16 + .../org.apache.kafka.connect.sink.SinkConnector | 16 + .../org.apache.kafka.connect.storage.Converter | 16 + ...rg.apache.kafka.connect.storage.HeaderConverter | 16 + .../org.apache.kafka.connect.storage.Converter | 16 + .../org.apache.kafka.connect.storage.Converter | 16 + .../java/kafka/log/remote/RemoteLogManager.java | 51 +- .../java/kafka/log/remote/RemoteLogReader.java | 12 +- core/src/main/scala/kafka/admin/AdminUtils.scala | 2 +- .../scala/kafka/admin/DeleteRecordsCommand.scala | 137 - .../scala/kafka/admin/LeaderElectionCommand.scala | 2 +- .../kafka/admin/ReassignPartitionsCommand.scala | 2 +- core/src/main/scala/kafka/admin/TopicCommand.scala | 5 +- core/src/main/scala/kafka/cluster/Partition.scala | 4 +- .../scala/kafka/controller/KafkaController.scala | 3 +- .../group/GroupCoordinatorAdapter.scala | 2 +- core/src/main/scala/kafka/log/UnifiedLog.scala | 24 +- .../src/main/scala/kafka/server/BrokerServer.scala | 9 +- .../main/scala/kafka/server/DelayedProduce.scala | 9 +- .../src/main/scala/kafka/server/FetchSession.scala | 21 + core/src/main/scala/kafka/server/KafkaApis.scala | 2 +- core/src/main/scala/kafka/server/KafkaConfig.scala | 14 +- .../scala/kafka/server/KafkaRequestHandler.scala | 30 + core/src/main/scala/kafka/server/KafkaServer.scala | 2 +- .../main/scala/kafka/server/ReplicaManager.scala | 3 +- .../main/scala/kafka/server/ZkAdminManager.scala | 3 +- .../main/scala/kafka/tools/ConsoleConsumer.scala | 1 - .../main/scala/kafka/tools/GetOffsetShell.scala | 91 +- core/src/main/scala/kafka/tools/MirrorMaker.scala | 4 +- .../kafka/tools/ReplicaVerificationTool.scala | 3 +- core/src/main/scala/kafka/utils/TopicFilter.scala | 55 - core/src/main/scala/kafka/zk/AdminZkClient.scala | 3 +- .../kafka/log/remote/RemoteLogManagerTest.java | 238 +- .../java/kafka/log/remote/RemoteLogReaderTest.java | 40 +- .../kafka/api/AuthorizerIntegrationTest.scala | 3 +- .../kafka/api/ProducerIdExpirationTest.scala | 6 +- .../scala/unit/kafka/admin/AddPartitionsTest.scala | 1 + .../admin/LeaderElectionCommandErrorTest.scala | 2 +- .../kafka/admin/LeaderElectionCommandTest.scala | 3 +- .../kafka/admin/ReassignPartitionsUnitTest.scala | 2 +- .../scala/unit/kafka/admin/TopicCommandTest.scala | 2 +- .../scala/unit/kafka/cluster/PartitionTest.scala | 8 +- .../group/GroupCoordinatorAdapterTest.scala | 4 +- .../unit/kafka/log/ProducerStateManagerTest.scala | 57 +- .../test/scala/unit/kafka/log/UnifiedLogTest.scala | 37 +- .../server/ConsumerGroupHeartbeatRequestTest.scala | 15 +- .../unit/kafka/server/DynamicConfigTest.scala | 2 +- .../scala/unit/kafka/server/KafkaApisTest.scala | 10 +- .../unit/kafka/server/ReplicaManagerTest.scala | 339 ++- .../scala/unit/kafka/server/RequestQuotaTest.scala | 4 +- .../test/scala/unit/kafka/utils/TestUtils.scala | 14 +- .../scala/unit/kafka/utils/TopicFilterTest.scala | 49 - .../scala/unit/kafka/zk/AdminZkClientTest.scala | 1 + docs/configuration.html | 47 +- docs/ops.html | 38 +- docs/toc.html | 3 +- gradle/dependencies.gradle | 6 +- gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 2 +- .../coordinator/group/GroupCoordinatorConfig.java | 42 +- .../coordinator/group/GroupCoordinatorService.java | 40 +- .../coordinator/group/GroupMetadataManager.java | 1504 ++++++++++- .../kafka/coordinator/group/OffsetAndMetadata.java | 125 + .../kafka/coordinator/group/RecordHelpers.java | 100 + .../group/ReplicatedGroupCoordinator.java | 46 +- .../coordinator/group/generic/GenericGroup.java | 111 +- .../group/generic/GenericGroupMember.java | 104 +- .../kafka/coordinator/group/generic/Protocol.java | 87 - .../group/runtime/CoordinatorBuilder.java | 14 +- .../group/runtime/CoordinatorResult.java | 70 +- .../group/runtime/CoordinatorRuntime.java | 17 +- .../group/runtime/CoordinatorTimer.java | 9 +- .../common/message/GroupMetadataValue.json | 2 +- .../group/GroupCoordinatorConfigTest.java | 12 +- .../group/GroupCoordinatorServiceTest.java | 129 +- .../group/GroupMetadataManagerTest.java | 2641 ++++++++++++++++++-- .../coordinator/group/MockCoordinatorTimer.java | 45 +- .../coordinator/group/OffsetAndMetadataTest.java | 74 + .../kafka/coordinator/group/RecordHelpersTest.java | 188 +- .../group/ReplicatedGroupCoordinatorTest.java | 37 + .../group/generic/GenericGroupMemberTest.java | 127 +- .../group/generic/GenericGroupTest.java | 328 +-- .../group/runtime/CoordinatorRuntimeTest.java | 40 +- .../apache/kafka/jmh/server/CheckpointBench.java | 2 +- licenses/classgraph-MIT | 21 - .../kafka/image/writer/ImageWriterOptions.java | 19 +- .../org/apache/kafka/image/ImageDowngradeTest.java | 2 +- .../kafka/image/writer/ImageWriterOptionsTest.java | 21 + release.py | 24 +- .../server/common/AdminCommandFailedException.java | 19 +- .../server/common/AdminOperationException.java | 21 +- .../kafka/server/common/MetadataVersion.java | 12 + .../java/org/apache/kafka/server/util/Json.java | 107 + .../apache/kafka/server/util/PartitionFilter.java | 66 + .../org/apache/kafka/server/util/TopicFilter.java | 73 + .../kafka/server/util/TopicPartitionFilter.java | 71 + .../apache/kafka/server/util/json/DecodeJson.java | 127 + .../apache/kafka/server/util/json/JsonArray.java | 67 + .../apache/kafka/server/util/json/JsonObject.java | 82 + .../apache/kafka/server/util/json/JsonValue.java | 130 + .../kafka/server/common/MetadataVersionTest.java | 20 + .../org/apache/kafka/server/util/JsonTest.java | 233 ++ .../apache/kafka/server/util/TopicFilterTest.java | 51 + .../remote/metadata/storage/ConsumerManager.java | 33 +- .../log/remote/metadata/storage/ConsumerTask.java | 15 +- .../storage/RemoteLogLeaderEpochState.java | 24 +- .../TopicBasedRemoteLogMetadataManager.java | 36 +- .../TopicBasedRemoteLogMetadataManagerConfig.java | 8 + .../kafka/storage/internals/log/LogFileUtils.java | 2 +- .../storage/internals/log/ProducerAppendInfo.java | 31 +- .../internals/log/ProducerStateManager.java | 26 +- .../internals/log/RemoteStorageThreadPool.java | 32 +- .../internals/log/VerificationStateEntry.java | 42 +- .../storage/RemoteLogLeaderEpochStateTest.java | 206 ++ .../storage/RemoteLogSegmentLifecycleTest.java | 8 +- ...picBasedRemoteLogMetadataManagerConfigTest.java | 13 +- ...icBasedRemoteLogMetadataManagerRestartTest.java | 2 +- .../TopicBasedRemoteLogMetadataManagerTest.java | 8 +- .../log/remote/storage/LocalTieredStorage.java | 34 +- .../log/remote/storage/LocalTieredStorageTest.java | 77 +- .../remote/storage/RemoteLogSegmentFileset.java | 34 +- .../streams/kstream/internals/KStreamImpl.java | 5 +- .../internals/graph/StreamTableJoinNode.java | 17 +- .../internals/InternalTopologyBuilder.java | 9 + .../processor/internals/assignment/Graph.java | 377 +++ .../{ => assignment}/RackAwareTaskAssignor.java | 3 +- .../integration/AbstractResetIntegrationTest.java | 2 +- .../streams/integration/ResetIntegrationTest.java | 10 +- .../kstream/internals/KStreamKTableJoinTest.java | 70 + .../processor/internals/TaskManagerTest.java | 97 +- .../processor/internals/assignment/GraphTest.java | 414 +++ .../assignment/RackAwareTaskAssignorTest.java | 1 - tests/kafkatest/directory_layout/kafka_path.py | 7 +- .../sanity_checks/test_verifiable_producer.py | 2 +- tests/kafkatest/services/verifiable_client.py | 9 +- .../apache/kafka/tools/ConsumerPerformance.java | 1 - .../apache/kafka/tools/DeleteRecordsCommand.java | 183 ++ .../apache/kafka/tools/MetadataQuorumCommand.java | 1 - .../apache/kafka/tools/ProducerPerformance.java | 2 +- .../org/apache/kafka/tools/StreamsResetter.java | 505 ++-- .../java/org/apache/kafka/tools}/ToolsUtils.java | 3 +- .../apache/kafka/tools/TransactionsCommand.java | 9 +- .../org/apache/kafka/tools/VerifiableProducer.java | 1 + .../kafka/tools/DeleteRecordsCommandTest.java | 182 ++ .../apache/kafka}/tools/StreamsResetterTest.java | 26 +- 217 files changed, 11061 insertions(+), 2123 deletions(-)
