This is an automated email from the ASF dual-hosted git repository.
bbejeck pushed a commit to branch Merge_AK_to_CCS_10_08_2020
in repository https://gitbox.apache.org/repos/asf/kafka.git
commit c257870501916a2cf371da488be189652801437d
Merge: 973f99e5 d252185
Author: Colin P. McCabe <cmcc...@apache.org>
AuthorDate: Tue Sep 8 13:03:29 2020 -0700
Merge remote-tracking branch 'apache/trunk' into master
README.md | 3 +-
bin/kafka-server-stop.sh | 5 +-
bin/zookeeper-server-stop.sh | 5 +-
build.gradle | 30 +-
checkstyle/import-control-jmh-benchmarks.xml | 2 +-
checkstyle/import-control.xml | 2 +
checkstyle/suppressions.xml | 6 +-
.../org/apache/kafka/clients/ClientRequest.java | 20 +-
.../java/org/apache/kafka/clients/KafkaClient.java | 6 +-
.../org/apache/kafka/clients/NetworkClient.java | 6 +-
.../java/org/apache/kafka/clients/admin/Admin.java | 92 +++
.../kafka/clients/admin/AdminClientConfig.java | 2 +-
.../admin/AlterUserScramCredentialsOptions.java | 21 +-
.../admin/AlterUserScramCredentialsResult.java | 60 ++
.../admin/DescribeUserScramCredentialsOptions.java | 21 +-
.../admin/DescribeUserScramCredentialsResult.java | 150 ++++
.../kafka/clients/admin/KafkaAdminClient.java | 192 ++++-
.../kafka/clients/admin/ScramCredentialInfo.java | 78 +++
.../apache/kafka/clients/admin/ScramMechanism.java | 81 +++
.../admin/UserScramCredentialAlteration.java | 33 +-
.../clients/admin/UserScramCredentialDeletion.java | 33 +-
.../admin/UserScramCredentialUpsertion.java | 100 +++
.../admin/UserScramCredentialsDescription.java | 82 +++
.../kafka/clients/consumer/ConsumerConfig.java | 2 +-
.../consumer/internals/ConsumerNetworkClient.java | 8 +-
.../kafka/clients/consumer/internals/Fetcher.java | 38 +-
.../consumer/internals/SubscriptionState.java | 66 +-
.../kafka/clients/producer/ProducerConfig.java | 2 +-
.../kafka/clients/producer/internals/Sender.java | 6 +-
.../config/provider/DirectoryConfigProvider.java | 106 +++
.../common/errors/DuplicateResourceException.java | 77 +++
.../common/errors/ResourceNotFoundException.java | 76 ++
.../errors/UnacceptableCredentialException.java | 29 +-
.../kafka/common/internals/PartitionStates.java | 11 +-
.../org/apache/kafka/common/metrics/Sensor.java | 12 +-
.../org/apache/kafka/common/protocol/ApiKeys.java | 10 +-
.../org/apache/kafka/common/protocol/Errors.java | 8 +-
.../org/apache/kafka/common/protocol/Message.java | 42 --
.../kafka/common/requests/AbstractRequest.java | 4 +
.../kafka/common/requests/AbstractResponse.java | 4 +
.../requests/AlterUserScramCredentialsRequest.java | 102 +++
.../AlterUserScramCredentialsResponse.java | 70 ++
.../DescribeUserScramCredentialsRequest.java | 87 +++
.../DescribeUserScramCredentialsResponse.java | 70 ++
.../kafka/common/requests/RequestContext.java | 14 +-
.../kafka/common/requests/RequestHeader.java | 31 +-
.../authenticator/SaslServerAuthenticator.java | 11 +-
.../security/scram/internals/ScramFormatter.java | 30 +-
.../security/scram/internals/ScramSaslClient.java | 4 +-
.../security/scram/internals/ScramSaslServer.java | 2 +-
.../authorizer/AuthorizableRequestContext.java | 9 +
.../message/AlterUserScramCredentialsRequest.json | 44 ++
.../message/AlterUserScramCredentialsResponse.json | 35 +
.../DescribeUserScramCredentialsRequest.json | 29 +
.../DescribeUserScramCredentialsResponse.json | 45 ++
.../resources/common/message/RequestHeader.json | 8 +-
.../java/org/apache/kafka/clients/MockClient.java | 6 +-
.../apache/kafka/clients/NetworkClientTest.java | 28 +-
.../DescribeUserScramCredentialsResultTest.java | 118 ++++
.../kafka/clients/admin/KafkaAdminClientTest.java | 154 +++++
.../kafka/clients/admin/MockAdminClient.java | 10 +
.../clients/consumer/internals/FetcherTest.java | 18 +-
.../clients/producer/internals/SenderTest.java | 4 +-
.../provider/DirectoryConfigProviderTest.java | 149 ++++
.../apache/kafka/common/message/MessageTest.java | 21 +-
.../apache/kafka/common/metrics/SensorTest.java | 59 +-
.../kafka/common/protocol/ProtoUtilsTest.java | 18 +-
.../kafka/common/requests/RequestContextTest.java | 12 +-
.../kafka/common/requests/RequestHeaderTest.java | 13 +
.../authenticator/SaslAuthenticatorTest.java | 2 +-
.../scram/internals/ScramFormatterTest.java | 12 +-
.../scram/internals/ScramMessagesTest.java | 4 +-
connect/mirror/README.md | 35 +-
.../src/main/scala/kafka/admin/ConfigCommand.scala | 163 +++--
core/src/main/scala/kafka/admin/TopicCommand.scala | 12 +-
.../scala/kafka/common/InterBrokerSendThread.scala | 2 +
.../scala/kafka/controller/KafkaController.scala | 9 +-
core/src/main/scala/kafka/log/LogSegment.scala | 2 +-
.../main/scala/kafka/network/RequestChannel.scala | 4 +-
.../main/scala/kafka/network/SocketServer.scala | 30 +-
.../src/main/scala/kafka/server/AdminManager.scala | 253 ++++++-
core/src/main/scala/kafka/server/KafkaApis.scala | 35 +-
.../main/scala/kafka/server/MetadataCache.scala | 2 +-
.../main/scala/kafka/tools/ConsoleProducer.scala | 2 +-
.../main/scala/kafka/utils/CommandLineUtils.scala | 5 +-
.../kafka/api/CustomQuotaCallbackTest.scala | 20 +-
.../DelegationTokenEndToEndAuthorizationTest.scala | 73 +-
.../kafka/api/EndToEndAuthorizationTest.scala | 20 +-
.../SaslClientsWithInvalidCredentialsTest.scala | 7 +-
.../SaslScramSslEndToEndAuthorizationTest.scala | 6 +-
.../scala/integration/kafka/api/SaslSetup.scala | 43 ++
.../server/DynamicBrokerReconfigurationTest.scala | 14 +-
.../kafka/common/InterBrokerSendThreadTest.scala | 12 +-
.../scala/unit/kafka/admin/ConfigCommandTest.scala | 125 +++-
.../admin/UserScramCredentialsCommandTest.scala | 137 ++++
.../controller/ControllerIntegrationTest.scala | 43 +-
.../kafka/integration/KafkaServerTestHarness.scala | 9 +
.../test/scala/unit/kafka/log/LogSegmentTest.scala | 56 +-
.../unit/kafka/network/RequestChannelTest.scala | 3 +-
.../unit/kafka/network/SocketServerTest.scala | 106 ++-
.../security/authorizer/AclAuthorizerTest.scala | 2 +-
.../AlterUserScramCredentialsRequestTest.scala | 433 ++++++++++++
.../kafka/server/BaseClientQuotaManagerTest.scala | 4 +-
.../kafka/server/ClientQuotasRequestTest.scala | 13 +-
.../DescribeUserScramCredentialsRequestTest.scala | 149 ++++
.../scala/unit/kafka/server/KafkaApisTest.scala | 6 +-
.../unit/kafka/server/LogDirFailureTest.scala | 38 +-
.../unit/kafka/server/MetadataCacheTest.scala | 49 +-
.../scala/unit/kafka/server/RequestQuotaTest.scala | 6 +
.../server/ThrottledChannelExpirationTest.scala | 2 +-
.../server/epoch/LeaderEpochFileCacheTest.scala | 6 +-
.../scala/unit/kafka/utils/JaasTestUtils.scala | 47 +-
.../test/scala/unit/kafka/utils/TestUtils.scala | 33 +-
docs/streams/developer-guide/dsl-api.html | 12 +-
docs/streams/upgrade-guide.html | 8 +
.../java/org/apache/kafka/message/FieldSpec.java | 340 +++++++++
.../java/org/apache/kafka/message/FieldType.java | 65 ++
.../kafka/message/JsonConverterGenerator.java | 412 +++++++++++
.../kafka/message/MessageClassGenerator.java | 26 +-
.../apache/kafka/message/MessageDataGenerator.java | 770 +++------------------
.../org/apache/kafka/message/MessageGenerator.java | 91 ++-
.../java/org/apache/kafka/message/MessageSpec.java | 2 +-
.../org/apache/kafka/message/SchemaGenerator.java | 2 +-
.../org/apache/kafka/message/StructRegistry.java | 74 +-
gradle/spotbugs-exclude.xml | 1 +
.../kafka/jmh/acl/AclAuthorizerBenchmark.java | 2 +-
.../jmh/consumer/SubscriptionStateBenchmark.java | 95 +++
.../jmh/metadata/MetadataRequestBenchmark.java | 3 +-
.../kafka/log4jappender/KafkaLog4jAppender.java | 22 +
.../streams/examples/wordcount/WordCountDemo.java | 28 +-
.../examples/wordcount/WordCountProcessorDemo.java | 25 +-
.../wordcount/WordCountTransformerDemo.java | 24 +-
.../examples/wordcount/WordCountDemoTest.java | 24 +-
.../org/apache/kafka/streams/StreamsBuilder.java | 64 +-
.../org/apache/kafka/streams/StreamsConfig.java | 3 +-
.../java/org/apache/kafka/streams/Topology.java | 5 +-
.../MissingSourceTopicException.java} | 18 +-
.../streams/errors/StreamsNotStartedException.java | 5 +-
.../kafka/streams/kstream/CogroupedKStream.java | 9 +
.../kafka/streams/kstream/KGroupedStream.java | 7 +
.../kafka/streams/kstream/SlidingWindows.java | 140 ++++
.../kstream/internals/CogroupedKStreamImpl.java | 14 +
.../internals/CogroupedStreamAggregateBuilder.java | 75 +-
.../kstream/internals/InternalStreamsBuilder.java | 21 +-
.../kstream/internals/KGroupedStreamImpl.java | 16 +
.../internals/KStreamSlidingWindowAggregate.java | 307 ++++++++
.../SessionWindowedCogroupedKStreamImpl.java | 42 +-
...va => SlidingWindowedCogroupedKStreamImpl.java} | 118 ++--
...amImpl.java => SlidingWindowedKStreamImpl.java} | 144 ++--
.../TimeWindowedCogroupedKStreamImpl.java | 52 +-
.../kstream/internals/TimeWindowedKStreamImpl.java | 45 +-
.../kstream/internals/graph/GlobalStoreNode.java | 12 +-
.../internals/graph/GraphGraceSearchUtil.java | 6 +
.../kstream/internals/graph/TableSourceNode.java | 19 +-
.../streams/processor/api/ProcessorContext.java | 3 +-
.../internals/AbstractReadOnlyDecorator.java | 46 +-
.../internals/AbstractReadWriteDecorator.java | 43 +-
.../internals/InternalTopologyBuilder.java | 35 +-
.../processor/internals/ProcessorAdapter.java | 2 +-
.../internals/ProcessorContextAdapter.java | 2 +-
.../streams/processor/internals/ProcessorNode.java | 16 +-
.../streams/processor/internals/StreamTask.java | 19 +-
.../streams/processor/internals/StreamThread.java | 2 +-
.../internals/StreamsRebalanceListener.java | 8 +-
.../internals/metrics/ProcessorNodeMetrics.java | 24 +
.../internals/metrics/StreamsMetricsImpl.java | 99 ++-
.../processor/internals/metrics/TaskMetrics.java | 30 +-
.../apache/kafka/streams/state/KeyValueStore.java | 4 +-
.../kafka/streams/state/ReadOnlyKeyValueStore.java | 53 +-
.../kafka/streams/state/ReadOnlyWindowStore.java | 173 +++--
.../apache/kafka/streams/state/WindowStore.java | 115 ++-
.../AbstractMergedSortedCacheStoreIterator.java | 70 +-
.../AbstractRocksDBSegmentedBytesStore.java | 87 ++-
.../streams/state/internals/AbstractSegments.java | 29 +-
.../state/internals/CachingKeyValueStore.java | 38 +-
.../state/internals/CachingSessionStore.java | 5 +-
.../state/internals/CachingWindowStore.java | 242 +++++--
.../internals/ChangeLoggingKeyValueBytesStore.java | 11 +
.../internals/ChangeLoggingWindowBytesStore.java | 31 +-
.../state/internals/CompositeKeyValueIterator.java | 3 +-
.../internals/CompositeReadOnlyKeyValueStore.java | 47 +-
.../internals/CompositeReadOnlyWindowStore.java | 112 ++-
.../state/internals/InMemoryKeyValueStore.java | 51 +-
.../state/internals/InMemorySessionStore.java | 5 +-
.../InMemoryTimeOrderedKeyValueBuffer.java | 2 +-
.../state/internals/InMemoryWindowStore.java | 182 +++--
.../streams/state/internals/KeyValueSegments.java | 2 +-
...ValueToTimestampedKeyValueByteStoreAdapter.java | 11 +
.../streams/state/internals/MemoryLRUCache.java | 16 +
.../state/internals/MemoryNavigableLRUCache.java | 28 +-
...ergedSortedCacheKeyValueBytesStoreIterator.java | 8 +-
.../MergedSortedCacheSessionStoreIterator.java | 2 +-
.../MergedSortedCacheWindowStoreIterator.java | 5 +-
...rgedSortedCacheWindowStoreKeyValueIterator.java | 5 +-
.../state/internals/MeteredKeyValueStore.java | 34 +-
.../state/internals/MeteredSessionStore.java | 15 +-
.../state/internals/MeteredWindowStore.java | 61 +-
.../kafka/streams/state/internals/NamedCache.java | 20 +-
.../internals/ReadOnlyKeyValueStoreFacade.java | 11 +
.../state/internals/ReadOnlyWindowStoreFacade.java | 58 +-
.../state/internals/RocksDBPrefixIterator.java | 54 --
.../state/internals/RocksDBRangeIterator.java | 40 +-
.../streams/state/internals/RocksDBStore.java | 54 +-
.../state/internals/RocksDBTimestampedStore.java | 106 ++-
.../state/internals/RocksDBWindowStore.java | 36 +-
.../streams/state/internals/RocksDbIterator.java | 8 +-
.../kafka/streams/state/internals/Segment.java | 4 -
.../streams/state/internals/SegmentIterator.java | 19 +-
.../state/internals/SegmentedBytesStore.java | 37 +-
.../kafka/streams/state/internals/Segments.java | 4 +-
.../streams/state/internals/SessionKeySchema.java | 5 +-
.../kafka/streams/state/internals/ThreadCache.java | 24 +-
.../internals/TimestampedKeyValueStoreBuilder.java | 11 +
.../state/internals/TimestampedSegments.java | 2 +-
.../internals/TimestampedWindowStoreBuilder.java | 42 +-
.../streams/state/internals/WindowKeySchema.java | 9 +-
.../WindowToTimestampedWindowByteStoreAdapter.java | 74 +-
.../state/internals/metrics/RocksDBMetrics.java | 374 +++++++++-
.../internals/metrics/RocksDBMetricsRecorder.java | 258 ++++++-
.../state/internals/metrics/StateStoreMetrics.java | 42 +-
.../apache/kafka/streams/StreamsBuilderTest.java | 258 ++++---
.../integration/AbstractResetIntegrationTest.java | 301 ++------
.../integration/GlobalKTableIntegrationTest.java | 16 +-
.../integration/GlobalThreadShutDownOrderTest.java | 6 +-
.../KStreamAggregationIntegrationTest.java | 204 ++++++
.../integration/MetricsIntegrationTest.java | 28 +-
.../streams/integration/ResetIntegrationTest.java | 241 ++++++-
.../integration/ResetIntegrationWithSslTest.java | 20 +-
.../integration/RocksDBMetricsIntegrationTest.java | 47 +-
.../kafka/streams/kstream/SessionWindowsTest.java | 9 +-
.../kafka/streams/kstream/SlidingWindowsTest.java | 82 +++
.../kstream/internals/AbstractStreamTest.java | 2 +-
.../internals/CogroupedKStreamImplTest.java | 55 +-
.../kstream/internals/GlobalKTableJoinsTest.java | 2 +-
.../kstream/internals/KGroupedStreamImplTest.java | 331 ++++++---
.../kstream/internals/KGroupedTableImplTest.java | 6 +-
.../kstream/internals/KStreamBranchTest.java | 6 +-
.../kstream/internals/KStreamFilterTest.java | 4 +-
.../kstream/internals/KStreamFlatMapTest.java | 4 +-
.../internals/KStreamFlatMapValuesTest.java | 4 +-
.../streams/kstream/internals/KStreamImplTest.java | 18 +-
.../streams/kstream/internals/KStreamMapTest.java | 4 +-
.../kstream/internals/KStreamMapValuesTest.java | 4 +-
.../kstream/internals/KStreamSelectKeyTest.java | 4 +-
.../KStreamSlidingWindowAggregateTest.java | 692 ++++++++++++++++++
.../kstream/internals/KStreamTransformTest.java | 8 +-
.../internals/KStreamTransformValuesTest.java | 4 +-
.../internals/KStreamWindowAggregateTest.java | 2 +-
.../kstream/internals/KTableAggregateTest.java | 8 +-
.../streams/kstream/internals/KTableImplTest.java | 8 +-
.../kstream/internals/KTableMapKeysTest.java | 4 +-
.../kstream/internals/KTableMapValuesTest.java | 2 +-
.../kstream/internals/KTableSourceTest.java | 2 +-
.../internals/KTableTransformValuesTest.java | 4 +-
.../internals/SessionWindowedKStreamImplTest.java | 6 +-
.../SlidingWindowedCogroupedKStreamImplTest.java | 249 +++++++
.../internals/SlidingWindowedKStreamImplTest.java | 439 ++++++++++++
.../kstream/internals/SuppressScenarioTest.java | 84 +++
.../internals/TimeWindowedKStreamImplTest.java | 18 +-
.../internals/GlobalStreamThreadTest.java | 2 +-
...HandlingSourceTopicDeletionIntegrationTest.java | 114 +++
.../internals/InternalTopologyBuilderTest.java | 37 +-
.../internals/ProcessorContextImplTest.java | 27 +-
.../processor/internals/PunctuationQueueTest.java | 40 +-
.../processor/internals/StreamTaskTest.java | 167 ++++-
.../internals/StreamsRebalanceListenerTest.java | 74 ++
.../internals/metrics/StreamsMetricsImplTest.java | 497 ++++++++-----
.../internals/metrics/TaskMetricsTest.java | 1 -
.../kafka/streams/state/NoOpWindowStore.java | 45 +-
.../state/internals/AbstractKeyValueStoreTest.java | 169 ++++-
.../internals/AbstractSessionBytesStoreTest.java | 17 +-
.../internals/AbstractWindowBytesStoreTest.java | 223 ++++--
.../state/internals/CachingKeyValueStoreTest.java | 154 +++--
.../state/internals/CachingSessionStoreTest.java | 3 +-
.../state/internals/CachingWindowStoreTest.java | 238 ++++++-
.../ChangeLoggingWindowBytesStoreTest.java | 25 +-
.../CompositeReadOnlyKeyValueStoreTest.java | 168 +++--
.../CompositeReadOnlyWindowStoreTest.java | 264 +++++--
.../DelegatingPeekingKeyValueIteratorTest.java | 19 +-
.../state/internals/InMemoryKeyValueStoreTest.java | 6 +-
.../state/internals/InMemoryWindowStoreTest.java | 25 +-
.../state/internals/KeyValueSegmentTest.java | 2 +-
.../state/internals/KeyValueSegmentsTest.java | 41 +-
...dSortedCacheKeyValueBytesStoreIteratorTest.java | 98 ++-
...dSortedCacheWrappedWindowStoreIteratorTest.java | 106 ++-
...acheWrappedWindowStoreKeyValueIteratorTest.java | 76 +-
.../state/internals/MeteredWindowStoreTest.java | 11 +-
.../state/internals/ReadOnlyWindowStoreStub.java | 207 +++++-
.../state/internals/RocksDBKeyValueStoreTest.java | 6 +-
.../streams/state/internals/RocksDBStoreTest.java | 98 ++-
.../internals/RocksDBTimestampedStoreTest.java | 65 ++
.../state/internals/SegmentIteratorTest.java | 90 ++-
.../streams/state/internals/ThreadCacheTest.java | 10 +-
.../TimestampedKeyValueStoreBuilderTest.java | 21 +-
.../state/internals/TimestampedSegmentTest.java | 2 +-
.../state/internals/TimestampedSegmentsTest.java | 41 +-
.../TimestampedWindowStoreBuilderTest.java | 17 +-
.../state/internals/WindowKeySchemaTest.java | 64 +-
.../metrics/RocksDBMetricsRecorderGaugesTest.java | 267 +++++++
.../metrics/RocksDBMetricsRecorderTest.java | 169 ++++-
.../internals/metrics/RocksDBMetricsTest.java | 275 +++++++-
.../internals/metrics/StateStoreMetricsTest.java | 48 +-
.../{MockProcessor.java => MockApiProcessor.java} | 81 ++-
.../kafka/test/MockApiProcessorSupplier.java | 69 ++
.../java/org/apache/kafka/test/MockProcessor.java | 90 +--
.../apache/kafka/test/SegmentedBytesStoreStub.java | 112 ---
.../org/apache/kafka/test/StreamsTestUtils.java | 7 +-
.../org/apache/kafka/streams/scala/Serdes.scala | 4 +
.../kafka/streams/scala/StreamsBuilder.scala | 21 +
.../streams/scala/kstream/KGroupedStream.scala | 3 +-
.../scala/kstream/SessionWindowedKStream.scala | 3 +-
.../scala/kstream/TimeWindowedKStream.scala | 3 +-
.../streams/scala/{ => serialization}/Serdes.scala | 50 +-
...bleJoinScalaIntegrationTestImplicitSerdes.scala | 19 +-
.../apache/kafka/streams/scala/TopologyTest.scala | 43 +-
.../apache/kafka/streams/scala/WordCountTest.scala | 11 +-
.../kafka/streams/scala/kstream/ConsumedTest.scala | 20 +-
.../kafka/streams/scala/kstream/GroupedTest.scala | 12 +-
.../kafka/streams/scala/kstream/JoinedTest.scala | 16 +-
.../kafka/streams/scala/kstream/KStreamTest.scala | 2 +-
.../kafka/streams/scala/kstream/KTableTest.scala | 2 +-
.../streams/scala/kstream/MaterializedTest.scala | 23 +-
.../kafka/streams/scala/kstream/ProducedTest.scala | 12 +-
.../streams/scala/kstream/RepartitionedTest.scala | 24 +-
.../streams/scala/kstream/StreamJoinedTest.scala | 22 +-
.../kafka/streams/internals/WindowStoreFacade.java | 25 +
tests/docker/ducker-ak | 2 +-
tests/kafkatest/services/kafka/kafka.py | 33 +-
.../kafkatest/services/security/security_config.py | 44 +-
.../templates/admin_client_as_broker_jaas.conf | 19 +
tests/kafkatest/tests/client/quota_test.py | 4 +-
tests/kafkatest/tests/core/upgrade_test.py | 2 +-
tests/kafkatest/version.py | 11 +-
333 files changed, 15083 insertions(+), 3903 deletions(-)
diff --cc bin/kafka-server-stop.sh
index 2dc3193,96f5e74..fd6b0b6
--- a/bin/kafka-server-stop.sh
+++ b/bin/kafka-server-stop.sh
@@@ -20,9 -21,10 +21,11 @@@ if [[ "$OSNAME" == "OS/390" ]]; the
JOBNAME="KAFKSTRT"
fi
PIDS=$(ps -A -o pid,jobname,comm | grep -i $JOBNAME | grep java | grep -v
grep | awk '{print $1}')
+ elif [[ "$OSNAME" == "OS400" ]]; then
+ PIDS=$(ps -af | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk
'{print $2}')
else
PIDS=$(ps ax | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk
'{print $1}')
+ PIDS_SUPPORT=$(ps ax | grep -i
'io\.confluent\.support\.metrics\.SupportedKafka' | grep java | grep -v grep |
awk '{print $1}')
fi
if [ -z "$PIDS" ]; then
