This is an automated email from the ASF dual-hosted git repository. bbejeck pushed a commit to branch Merge_AK_to_CCS_10_08_2020 in repository https://gitbox.apache.org/repos/asf/kafka.git
commit c257870501916a2cf371da488be189652801437d Merge: 973f99e5 d252185 Author: Colin P. McCabe <cmcc...@apache.org> AuthorDate: Tue Sep 8 13:03:29 2020 -0700 Merge remote-tracking branch 'apache/trunk' into master README.md | 3 +- bin/kafka-server-stop.sh | 5 +- bin/zookeeper-server-stop.sh | 5 +- build.gradle | 30 +- checkstyle/import-control-jmh-benchmarks.xml | 2 +- checkstyle/import-control.xml | 2 + checkstyle/suppressions.xml | 6 +- .../org/apache/kafka/clients/ClientRequest.java | 20 +- .../java/org/apache/kafka/clients/KafkaClient.java | 6 +- .../org/apache/kafka/clients/NetworkClient.java | 6 +- .../java/org/apache/kafka/clients/admin/Admin.java | 92 +++ .../kafka/clients/admin/AdminClientConfig.java | 2 +- .../admin/AlterUserScramCredentialsOptions.java | 21 +- .../admin/AlterUserScramCredentialsResult.java | 60 ++ .../admin/DescribeUserScramCredentialsOptions.java | 21 +- .../admin/DescribeUserScramCredentialsResult.java | 150 ++++ .../kafka/clients/admin/KafkaAdminClient.java | 192 ++++- .../kafka/clients/admin/ScramCredentialInfo.java | 78 +++ .../apache/kafka/clients/admin/ScramMechanism.java | 81 +++ .../admin/UserScramCredentialAlteration.java | 33 +- .../clients/admin/UserScramCredentialDeletion.java | 33 +- .../admin/UserScramCredentialUpsertion.java | 100 +++ .../admin/UserScramCredentialsDescription.java | 82 +++ .../kafka/clients/consumer/ConsumerConfig.java | 2 +- .../consumer/internals/ConsumerNetworkClient.java | 8 +- .../kafka/clients/consumer/internals/Fetcher.java | 38 +- .../consumer/internals/SubscriptionState.java | 66 +- .../kafka/clients/producer/ProducerConfig.java | 2 +- .../kafka/clients/producer/internals/Sender.java | 6 +- .../config/provider/DirectoryConfigProvider.java | 106 +++ .../common/errors/DuplicateResourceException.java | 77 +++ .../common/errors/ResourceNotFoundException.java | 76 ++ .../errors/UnacceptableCredentialException.java | 29 +- .../kafka/common/internals/PartitionStates.java | 11 +- .../org/apache/kafka/common/metrics/Sensor.java | 12 +- .../org/apache/kafka/common/protocol/ApiKeys.java | 10 +- .../org/apache/kafka/common/protocol/Errors.java | 8 +- .../org/apache/kafka/common/protocol/Message.java | 42 -- .../kafka/common/requests/AbstractRequest.java | 4 + .../kafka/common/requests/AbstractResponse.java | 4 + .../requests/AlterUserScramCredentialsRequest.java | 102 +++ .../AlterUserScramCredentialsResponse.java | 70 ++ .../DescribeUserScramCredentialsRequest.java | 87 +++ .../DescribeUserScramCredentialsResponse.java | 70 ++ .../kafka/common/requests/RequestContext.java | 14 +- .../kafka/common/requests/RequestHeader.java | 31 +- .../authenticator/SaslServerAuthenticator.java | 11 +- .../security/scram/internals/ScramFormatter.java | 30 +- .../security/scram/internals/ScramSaslClient.java | 4 +- .../security/scram/internals/ScramSaslServer.java | 2 +- .../authorizer/AuthorizableRequestContext.java | 9 + .../message/AlterUserScramCredentialsRequest.json | 44 ++ .../message/AlterUserScramCredentialsResponse.json | 35 + .../DescribeUserScramCredentialsRequest.json | 29 + .../DescribeUserScramCredentialsResponse.json | 45 ++ .../resources/common/message/RequestHeader.json | 8 +- .../java/org/apache/kafka/clients/MockClient.java | 6 +- .../apache/kafka/clients/NetworkClientTest.java | 28 +- .../DescribeUserScramCredentialsResultTest.java | 118 ++++ .../kafka/clients/admin/KafkaAdminClientTest.java | 154 +++++ .../kafka/clients/admin/MockAdminClient.java | 10 + .../clients/consumer/internals/FetcherTest.java | 18 +- .../clients/producer/internals/SenderTest.java | 4 +- .../provider/DirectoryConfigProviderTest.java | 149 ++++ .../apache/kafka/common/message/MessageTest.java | 21 +- .../apache/kafka/common/metrics/SensorTest.java | 59 +- .../kafka/common/protocol/ProtoUtilsTest.java | 18 +- .../kafka/common/requests/RequestContextTest.java | 12 +- .../kafka/common/requests/RequestHeaderTest.java | 13 + .../authenticator/SaslAuthenticatorTest.java | 2 +- .../scram/internals/ScramFormatterTest.java | 12 +- .../scram/internals/ScramMessagesTest.java | 4 +- connect/mirror/README.md | 35 +- .../src/main/scala/kafka/admin/ConfigCommand.scala | 163 +++-- core/src/main/scala/kafka/admin/TopicCommand.scala | 12 +- .../scala/kafka/common/InterBrokerSendThread.scala | 2 + .../scala/kafka/controller/KafkaController.scala | 9 +- core/src/main/scala/kafka/log/LogSegment.scala | 2 +- .../main/scala/kafka/network/RequestChannel.scala | 4 +- .../main/scala/kafka/network/SocketServer.scala | 30 +- .../src/main/scala/kafka/server/AdminManager.scala | 253 ++++++- core/src/main/scala/kafka/server/KafkaApis.scala | 35 +- .../main/scala/kafka/server/MetadataCache.scala | 2 +- .../main/scala/kafka/tools/ConsoleProducer.scala | 2 +- .../main/scala/kafka/utils/CommandLineUtils.scala | 5 +- .../kafka/api/CustomQuotaCallbackTest.scala | 20 +- .../DelegationTokenEndToEndAuthorizationTest.scala | 73 +- .../kafka/api/EndToEndAuthorizationTest.scala | 20 +- .../SaslClientsWithInvalidCredentialsTest.scala | 7 +- .../SaslScramSslEndToEndAuthorizationTest.scala | 6 +- .../scala/integration/kafka/api/SaslSetup.scala | 43 ++ .../server/DynamicBrokerReconfigurationTest.scala | 14 +- .../kafka/common/InterBrokerSendThreadTest.scala | 12 +- .../scala/unit/kafka/admin/ConfigCommandTest.scala | 125 +++- .../admin/UserScramCredentialsCommandTest.scala | 137 ++++ .../controller/ControllerIntegrationTest.scala | 43 +- .../kafka/integration/KafkaServerTestHarness.scala | 9 + .../test/scala/unit/kafka/log/LogSegmentTest.scala | 56 +- .../unit/kafka/network/RequestChannelTest.scala | 3 +- .../unit/kafka/network/SocketServerTest.scala | 106 ++- .../security/authorizer/AclAuthorizerTest.scala | 2 +- .../AlterUserScramCredentialsRequestTest.scala | 433 ++++++++++++ .../kafka/server/BaseClientQuotaManagerTest.scala | 4 +- .../kafka/server/ClientQuotasRequestTest.scala | 13 +- .../DescribeUserScramCredentialsRequestTest.scala | 149 ++++ .../scala/unit/kafka/server/KafkaApisTest.scala | 6 +- .../unit/kafka/server/LogDirFailureTest.scala | 38 +- .../unit/kafka/server/MetadataCacheTest.scala | 49 +- .../scala/unit/kafka/server/RequestQuotaTest.scala | 6 + .../server/ThrottledChannelExpirationTest.scala | 2 +- .../server/epoch/LeaderEpochFileCacheTest.scala | 6 +- .../scala/unit/kafka/utils/JaasTestUtils.scala | 47 +- .../test/scala/unit/kafka/utils/TestUtils.scala | 33 +- docs/streams/developer-guide/dsl-api.html | 12 +- docs/streams/upgrade-guide.html | 8 + .../java/org/apache/kafka/message/FieldSpec.java | 340 +++++++++ .../java/org/apache/kafka/message/FieldType.java | 65 ++ .../kafka/message/JsonConverterGenerator.java | 412 +++++++++++ .../kafka/message/MessageClassGenerator.java | 26 +- .../apache/kafka/message/MessageDataGenerator.java | 770 +++------------------ .../org/apache/kafka/message/MessageGenerator.java | 91 ++- .../java/org/apache/kafka/message/MessageSpec.java | 2 +- .../org/apache/kafka/message/SchemaGenerator.java | 2 +- .../org/apache/kafka/message/StructRegistry.java | 74 +- gradle/spotbugs-exclude.xml | 1 + .../kafka/jmh/acl/AclAuthorizerBenchmark.java | 2 +- .../jmh/consumer/SubscriptionStateBenchmark.java | 95 +++ .../jmh/metadata/MetadataRequestBenchmark.java | 3 +- .../kafka/log4jappender/KafkaLog4jAppender.java | 22 + .../streams/examples/wordcount/WordCountDemo.java | 28 +- .../examples/wordcount/WordCountProcessorDemo.java | 25 +- .../wordcount/WordCountTransformerDemo.java | 24 +- .../examples/wordcount/WordCountDemoTest.java | 24 +- .../org/apache/kafka/streams/StreamsBuilder.java | 64 +- .../org/apache/kafka/streams/StreamsConfig.java | 3 +- .../java/org/apache/kafka/streams/Topology.java | 5 +- .../MissingSourceTopicException.java} | 18 +- .../streams/errors/StreamsNotStartedException.java | 5 +- .../kafka/streams/kstream/CogroupedKStream.java | 9 + .../kafka/streams/kstream/KGroupedStream.java | 7 + .../kafka/streams/kstream/SlidingWindows.java | 140 ++++ .../kstream/internals/CogroupedKStreamImpl.java | 14 + .../internals/CogroupedStreamAggregateBuilder.java | 75 +- .../kstream/internals/InternalStreamsBuilder.java | 21 +- .../kstream/internals/KGroupedStreamImpl.java | 16 + .../internals/KStreamSlidingWindowAggregate.java | 307 ++++++++ .../SessionWindowedCogroupedKStreamImpl.java | 42 +- ...va => SlidingWindowedCogroupedKStreamImpl.java} | 118 ++-- ...amImpl.java => SlidingWindowedKStreamImpl.java} | 144 ++-- .../TimeWindowedCogroupedKStreamImpl.java | 52 +- .../kstream/internals/TimeWindowedKStreamImpl.java | 45 +- .../kstream/internals/graph/GlobalStoreNode.java | 12 +- .../internals/graph/GraphGraceSearchUtil.java | 6 + .../kstream/internals/graph/TableSourceNode.java | 19 +- .../streams/processor/api/ProcessorContext.java | 3 +- .../internals/AbstractReadOnlyDecorator.java | 46 +- .../internals/AbstractReadWriteDecorator.java | 43 +- .../internals/InternalTopologyBuilder.java | 35 +- .../processor/internals/ProcessorAdapter.java | 2 +- .../internals/ProcessorContextAdapter.java | 2 +- .../streams/processor/internals/ProcessorNode.java | 16 +- .../streams/processor/internals/StreamTask.java | 19 +- .../streams/processor/internals/StreamThread.java | 2 +- .../internals/StreamsRebalanceListener.java | 8 +- .../internals/metrics/ProcessorNodeMetrics.java | 24 + .../internals/metrics/StreamsMetricsImpl.java | 99 ++- .../processor/internals/metrics/TaskMetrics.java | 30 +- .../apache/kafka/streams/state/KeyValueStore.java | 4 +- .../kafka/streams/state/ReadOnlyKeyValueStore.java | 53 +- .../kafka/streams/state/ReadOnlyWindowStore.java | 173 +++-- .../apache/kafka/streams/state/WindowStore.java | 115 ++- .../AbstractMergedSortedCacheStoreIterator.java | 70 +- .../AbstractRocksDBSegmentedBytesStore.java | 87 ++- .../streams/state/internals/AbstractSegments.java | 29 +- .../state/internals/CachingKeyValueStore.java | 38 +- .../state/internals/CachingSessionStore.java | 5 +- .../state/internals/CachingWindowStore.java | 242 +++++-- .../internals/ChangeLoggingKeyValueBytesStore.java | 11 + .../internals/ChangeLoggingWindowBytesStore.java | 31 +- .../state/internals/CompositeKeyValueIterator.java | 3 +- .../internals/CompositeReadOnlyKeyValueStore.java | 47 +- .../internals/CompositeReadOnlyWindowStore.java | 112 ++- .../state/internals/InMemoryKeyValueStore.java | 51 +- .../state/internals/InMemorySessionStore.java | 5 +- .../InMemoryTimeOrderedKeyValueBuffer.java | 2 +- .../state/internals/InMemoryWindowStore.java | 182 +++-- .../streams/state/internals/KeyValueSegments.java | 2 +- ...ValueToTimestampedKeyValueByteStoreAdapter.java | 11 + .../streams/state/internals/MemoryLRUCache.java | 16 + .../state/internals/MemoryNavigableLRUCache.java | 28 +- ...ergedSortedCacheKeyValueBytesStoreIterator.java | 8 +- .../MergedSortedCacheSessionStoreIterator.java | 2 +- .../MergedSortedCacheWindowStoreIterator.java | 5 +- ...rgedSortedCacheWindowStoreKeyValueIterator.java | 5 +- .../state/internals/MeteredKeyValueStore.java | 34 +- .../state/internals/MeteredSessionStore.java | 15 +- .../state/internals/MeteredWindowStore.java | 61 +- .../kafka/streams/state/internals/NamedCache.java | 20 +- .../internals/ReadOnlyKeyValueStoreFacade.java | 11 + .../state/internals/ReadOnlyWindowStoreFacade.java | 58 +- .../state/internals/RocksDBPrefixIterator.java | 54 -- .../state/internals/RocksDBRangeIterator.java | 40 +- .../streams/state/internals/RocksDBStore.java | 54 +- .../state/internals/RocksDBTimestampedStore.java | 106 ++- .../state/internals/RocksDBWindowStore.java | 36 +- .../streams/state/internals/RocksDbIterator.java | 8 +- .../kafka/streams/state/internals/Segment.java | 4 - .../streams/state/internals/SegmentIterator.java | 19 +- .../state/internals/SegmentedBytesStore.java | 37 +- .../kafka/streams/state/internals/Segments.java | 4 +- .../streams/state/internals/SessionKeySchema.java | 5 +- .../kafka/streams/state/internals/ThreadCache.java | 24 +- .../internals/TimestampedKeyValueStoreBuilder.java | 11 + .../state/internals/TimestampedSegments.java | 2 +- .../internals/TimestampedWindowStoreBuilder.java | 42 +- .../streams/state/internals/WindowKeySchema.java | 9 +- .../WindowToTimestampedWindowByteStoreAdapter.java | 74 +- .../state/internals/metrics/RocksDBMetrics.java | 374 +++++++++- .../internals/metrics/RocksDBMetricsRecorder.java | 258 ++++++- .../state/internals/metrics/StateStoreMetrics.java | 42 +- .../apache/kafka/streams/StreamsBuilderTest.java | 258 ++++--- .../integration/AbstractResetIntegrationTest.java | 301 ++------ .../integration/GlobalKTableIntegrationTest.java | 16 +- .../integration/GlobalThreadShutDownOrderTest.java | 6 +- .../KStreamAggregationIntegrationTest.java | 204 ++++++ .../integration/MetricsIntegrationTest.java | 28 +- .../streams/integration/ResetIntegrationTest.java | 241 ++++++- .../integration/ResetIntegrationWithSslTest.java | 20 +- .../integration/RocksDBMetricsIntegrationTest.java | 47 +- .../kafka/streams/kstream/SessionWindowsTest.java | 9 +- .../kafka/streams/kstream/SlidingWindowsTest.java | 82 +++ .../kstream/internals/AbstractStreamTest.java | 2 +- .../internals/CogroupedKStreamImplTest.java | 55 +- .../kstream/internals/GlobalKTableJoinsTest.java | 2 +- .../kstream/internals/KGroupedStreamImplTest.java | 331 ++++++--- .../kstream/internals/KGroupedTableImplTest.java | 6 +- .../kstream/internals/KStreamBranchTest.java | 6 +- .../kstream/internals/KStreamFilterTest.java | 4 +- .../kstream/internals/KStreamFlatMapTest.java | 4 +- .../internals/KStreamFlatMapValuesTest.java | 4 +- .../streams/kstream/internals/KStreamImplTest.java | 18 +- .../streams/kstream/internals/KStreamMapTest.java | 4 +- .../kstream/internals/KStreamMapValuesTest.java | 4 +- .../kstream/internals/KStreamSelectKeyTest.java | 4 +- .../KStreamSlidingWindowAggregateTest.java | 692 ++++++++++++++++++ .../kstream/internals/KStreamTransformTest.java | 8 +- .../internals/KStreamTransformValuesTest.java | 4 +- .../internals/KStreamWindowAggregateTest.java | 2 +- .../kstream/internals/KTableAggregateTest.java | 8 +- .../streams/kstream/internals/KTableImplTest.java | 8 +- .../kstream/internals/KTableMapKeysTest.java | 4 +- .../kstream/internals/KTableMapValuesTest.java | 2 +- .../kstream/internals/KTableSourceTest.java | 2 +- .../internals/KTableTransformValuesTest.java | 4 +- .../internals/SessionWindowedKStreamImplTest.java | 6 +- .../SlidingWindowedCogroupedKStreamImplTest.java | 249 +++++++ .../internals/SlidingWindowedKStreamImplTest.java | 439 ++++++++++++ .../kstream/internals/SuppressScenarioTest.java | 84 +++ .../internals/TimeWindowedKStreamImplTest.java | 18 +- .../internals/GlobalStreamThreadTest.java | 2 +- ...HandlingSourceTopicDeletionIntegrationTest.java | 114 +++ .../internals/InternalTopologyBuilderTest.java | 37 +- .../internals/ProcessorContextImplTest.java | 27 +- .../processor/internals/PunctuationQueueTest.java | 40 +- .../processor/internals/StreamTaskTest.java | 167 ++++- .../internals/StreamsRebalanceListenerTest.java | 74 ++ .../internals/metrics/StreamsMetricsImplTest.java | 497 ++++++++----- .../internals/metrics/TaskMetricsTest.java | 1 - .../kafka/streams/state/NoOpWindowStore.java | 45 +- .../state/internals/AbstractKeyValueStoreTest.java | 169 ++++- .../internals/AbstractSessionBytesStoreTest.java | 17 +- .../internals/AbstractWindowBytesStoreTest.java | 223 ++++-- .../state/internals/CachingKeyValueStoreTest.java | 154 +++-- .../state/internals/CachingSessionStoreTest.java | 3 +- .../state/internals/CachingWindowStoreTest.java | 238 ++++++- .../ChangeLoggingWindowBytesStoreTest.java | 25 +- .../CompositeReadOnlyKeyValueStoreTest.java | 168 +++-- .../CompositeReadOnlyWindowStoreTest.java | 264 +++++-- .../DelegatingPeekingKeyValueIteratorTest.java | 19 +- .../state/internals/InMemoryKeyValueStoreTest.java | 6 +- .../state/internals/InMemoryWindowStoreTest.java | 25 +- .../state/internals/KeyValueSegmentTest.java | 2 +- .../state/internals/KeyValueSegmentsTest.java | 41 +- ...dSortedCacheKeyValueBytesStoreIteratorTest.java | 98 ++- ...dSortedCacheWrappedWindowStoreIteratorTest.java | 106 ++- ...acheWrappedWindowStoreKeyValueIteratorTest.java | 76 +- .../state/internals/MeteredWindowStoreTest.java | 11 +- .../state/internals/ReadOnlyWindowStoreStub.java | 207 +++++- .../state/internals/RocksDBKeyValueStoreTest.java | 6 +- .../streams/state/internals/RocksDBStoreTest.java | 98 ++- .../internals/RocksDBTimestampedStoreTest.java | 65 ++ .../state/internals/SegmentIteratorTest.java | 90 ++- .../streams/state/internals/ThreadCacheTest.java | 10 +- .../TimestampedKeyValueStoreBuilderTest.java | 21 +- .../state/internals/TimestampedSegmentTest.java | 2 +- .../state/internals/TimestampedSegmentsTest.java | 41 +- .../TimestampedWindowStoreBuilderTest.java | 17 +- .../state/internals/WindowKeySchemaTest.java | 64 +- .../metrics/RocksDBMetricsRecorderGaugesTest.java | 267 +++++++ .../metrics/RocksDBMetricsRecorderTest.java | 169 ++++- .../internals/metrics/RocksDBMetricsTest.java | 275 +++++++- .../internals/metrics/StateStoreMetricsTest.java | 48 +- .../{MockProcessor.java => MockApiProcessor.java} | 81 ++- .../kafka/test/MockApiProcessorSupplier.java | 69 ++ .../java/org/apache/kafka/test/MockProcessor.java | 90 +-- .../apache/kafka/test/SegmentedBytesStoreStub.java | 112 --- .../org/apache/kafka/test/StreamsTestUtils.java | 7 +- .../org/apache/kafka/streams/scala/Serdes.scala | 4 + .../kafka/streams/scala/StreamsBuilder.scala | 21 + .../streams/scala/kstream/KGroupedStream.scala | 3 +- .../scala/kstream/SessionWindowedKStream.scala | 3 +- .../scala/kstream/TimeWindowedKStream.scala | 3 +- .../streams/scala/{ => serialization}/Serdes.scala | 50 +- ...bleJoinScalaIntegrationTestImplicitSerdes.scala | 19 +- .../apache/kafka/streams/scala/TopologyTest.scala | 43 +- .../apache/kafka/streams/scala/WordCountTest.scala | 11 +- .../kafka/streams/scala/kstream/ConsumedTest.scala | 20 +- .../kafka/streams/scala/kstream/GroupedTest.scala | 12 +- .../kafka/streams/scala/kstream/JoinedTest.scala | 16 +- .../kafka/streams/scala/kstream/KStreamTest.scala | 2 +- .../kafka/streams/scala/kstream/KTableTest.scala | 2 +- .../streams/scala/kstream/MaterializedTest.scala | 23 +- .../kafka/streams/scala/kstream/ProducedTest.scala | 12 +- .../streams/scala/kstream/RepartitionedTest.scala | 24 +- .../streams/scala/kstream/StreamJoinedTest.scala | 22 +- .../kafka/streams/internals/WindowStoreFacade.java | 25 + tests/docker/ducker-ak | 2 +- tests/kafkatest/services/kafka/kafka.py | 33 +- .../kafkatest/services/security/security_config.py | 44 +- .../templates/admin_client_as_broker_jaas.conf | 19 + tests/kafkatest/tests/client/quota_test.py | 4 +- tests/kafkatest/tests/core/upgrade_test.py | 2 +- tests/kafkatest/version.py | 11 +- 333 files changed, 15083 insertions(+), 3903 deletions(-) diff --cc bin/kafka-server-stop.sh index 2dc3193,96f5e74..fd6b0b6 --- a/bin/kafka-server-stop.sh +++ b/bin/kafka-server-stop.sh @@@ -20,9 -21,10 +21,11 @@@ if [[ "$OSNAME" == "OS/390" ]]; the JOBNAME="KAFKSTRT" fi PIDS=$(ps -A -o pid,jobname,comm | grep -i $JOBNAME | grep java | grep -v grep | awk '{print $1}') + elif [[ "$OSNAME" == "OS400" ]]; then + PIDS=$(ps -af | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $2}') else PIDS=$(ps ax | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}') + PIDS_SUPPORT=$(ps ax | grep -i 'io\.confluent\.support\.metrics\.SupportedKafka' | grep java | grep -v grep | awk '{print $1}') fi if [ -z "$PIDS" ]; then