This is an automated email from the ASF dual-hosted git repository.
fpaul pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 4f20772 [FLINK-26148][runtime] Change the format of adaptive batch
scheduler config option to jobmanager.adaptive-batch-scheduler.XXX
new 1602e4b [FLINK-24246][connector/pulsar] Bump PulsarClient version to
latest 2.9.1
new 36de46d [FLINK-26020][connector/pulsar] Unified Pulsar Connector
config model for Pulsar source and sink.
new a195f72 [FLINK-26021][connector/pulsar] Add the ability to merge the
partitioned Pulsar topics.
new 9bc8b0f [FLINK-26023][connector/pulsar] Create a Pulsar sink config
model for matching ProducerConfigurationData.
new 0e72bfed [FLINK-26024][connector/pulsar] Create a
PulsarSerializationSchema for better records serialization.
new 136add5 [FLINK-26022][connector/pulsar] Implement at-least-once and
exactly-once Pulsar Sink.
new b6be14d [FLINK-26025][connector/pulsar] Replace MockPulsar with new
Pulsar test tools based on PulsarStandalone.
new 714dd80 [FLINK-26026][connector/pulsar] Create unit tests for Pulsar
sink connector.
new 07f23e0 [FLINK-26038][connector/pulsar] Support delay message on
PulsarSink.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../generated/pulsar_client_configuration.html | 8 +-
.../generated/pulsar_source_configuration.html | 2 +-
.../5b9eed8a-5fb6-4373-98ac-3be2a71941b8 | 11 -
flink-connectors/flink-connector-pulsar/pom.xml | 90 +++--
...arConfigUtils.java => PulsarClientFactory.java} | 209 ++++-------
.../pulsar/common/config/PulsarConfigBuilder.java | 143 +++++++
.../common/config/PulsarConfigValidator.java | 105 ++++++
.../pulsar/common/config/PulsarConfiguration.java | 104 ++++++
.../pulsar/common/config/PulsarOptions.java | 18 +-
.../common/utils/PulsarTransactionUtils.java | 108 ++----
.../flink/connector/pulsar/sink/PulsarSink.java | 145 ++++++++
.../connector/pulsar/sink/PulsarSinkBuilder.java | 372 ++++++++++++++++++
.../connector/pulsar/sink/PulsarSinkOptions.java | 269 +++++++++++++
.../pulsar/sink/committer/PulsarCommittable.java | 71 ++++
.../committer/PulsarCommittableSerializer.java | 65 ++++
.../pulsar/sink/committer/PulsarCommitter.java | 174 +++++++++
.../pulsar/sink/config/PulsarSinkConfigUtils.java | 112 ++++++
.../pulsar/sink/config/SinkConfiguration.java | 160 ++++++++
.../connector/pulsar/sink/writer/PulsarWriter.java | 274 ++++++++++++++
.../sink/writer/context/PulsarSinkContext.java | 46 +++
.../sink/writer/context/PulsarSinkContextImpl.java | 61 +++
.../writer/delayer/FixedMessageDelayer.java} | 25 +-
.../pulsar/sink/writer/delayer/MessageDelayer.java | 62 +++
.../pulsar/sink/writer/message/PulsarMessage.java | 111 ++++++
.../sink/writer/message/PulsarMessageBuilder.java | 127 +++++++
.../sink/writer/router/KeyHashTopicRouter.java | 71 ++++
.../pulsar/sink/writer/router/MessageKeyHash.java | 85 +++++
.../sink/writer/router/RoundRobinTopicRouter.java | 63 ++++
.../pulsar/sink/writer/router/TopicRouter.java | 64 ++++
.../sink/writer/router/TopicRoutingMode.java | 87 +++++
.../writer/serializer/PulsarSchemaWrapper.java | 59 +++
.../serializer/PulsarSerializationSchema.java | 129 +++++++
.../PulsarSerializationSchemaWrapper.java | 59 +++
.../sink/writer/topic/TopicMetadataListener.java | 173 +++++++++
.../sink/writer/topic/TopicProducerRegister.java | 202 ++++++++++
.../connector/pulsar/source/PulsarSource.java | 27 +-
.../pulsar/source/PulsarSourceBuilder.java | 122 +++---
.../pulsar/source/PulsarSourceOptions.java | 12 +-
.../pulsar/source/config/CursorVerification.java | 23 +-
.../source/config/PulsarSourceConfigUtils.java | 138 +++----
.../pulsar/source/config/SourceConfiguration.java | 190 ++++++----
.../source/enumerator/PulsarSourceEnumerator.java | 18 +-
.../source/enumerator/SplitsAssignmentState.java | 2 +-
.../cursor/stop/LatestMessageStopCursor.java | 1 +
.../source/enumerator/topic/TopicNameUtils.java | 45 +++
.../source/enumerator/topic/TopicPartition.java | 4 +-
.../enumerator/topic/range/RangeGenerator.java | 8 +
.../source/reader/PulsarSourceReaderFactory.java | 19 +-
.../deserializer/PulsarDeserializationSchema.java | 9 +
.../PulsarDeserializationSchemaWrapper.java | 4 +-
.../reader/deserializer/PulsarSchemaWrapper.java | 12 +-
.../reader/source/PulsarOrderedSourceReader.java | 5 +-
.../reader/source/PulsarSourceReaderBase.java | 4 +-
.../reader/source/PulsarUnorderedSourceReader.java | 3 -
.../split/PulsarOrderedPartitionSplitReader.java | 4 +-
.../split/PulsarPartitionSplitReaderBase.java | 6 +-
.../split/PulsarUnorderedPartitionSplitReader.java | 23 +-
.../common/config/PulsarConfigBuilderTest.java | 76 ++++
.../common/config/PulsarConfigValidatorTest.java | 57 +++
.../common/config/PulsarConfigurationTest.java | 65 ++++
.../common/schema/PulsarSchemaUtilsTest.java | 6 +-
.../pulsar/sink/PulsarSinkBuilderTest.java | 107 ++++++
.../connector/pulsar/sink/PulsarSinkITCase.java | 99 +++++
.../committer/PulsarCommittableSerializerTest.java | 53 +++
.../pulsar/sink/writer/PulsarWriterTest.java | 202 ++++++++++
.../sink/writer/router/KeyHashTopicRouterTest.java | 111 ++++++
.../writer/router/RoundRobinTopicRouterTest.java | 88 +++++
.../writer/topic/TopicMetadataListenerTest.java | 140 +++++++
.../writer/topic/TopicProducerRegisterTest.java | 91 +++++
.../pulsar/source/PulsarSourceBuilderTest.java | 69 ++--
.../pulsar/source/PulsarSourceITCase.java | 2 +-
.../enumerator/PulsarSourceEnumeratorTest.java | 1 -
.../subscriber/PulsarSubscriberTest.java | 10 +-
.../enumerator/topic/TopicNameUtilsTest.java | 16 +
.../PulsarDeserializationSchemaTest.java | 10 +-
.../reader/source/PulsarSourceReaderTestBase.java | 8 +-
.../split/PulsarPartitionSplitReaderTestBase.java | 11 +-
.../pulsar/testutils/PulsarTestContext.java | 4 -
.../pulsar/testutils/PulsarTestSuiteBase.java | 2 +-
.../connector/pulsar/testutils/SampleData.java | 96 -----
.../cases/MultipleTopicConsumingContext.java | 1 -
.../cases/MultipleTopicTemplateContext.java | 1 -
.../cases/SingleTopicConsumingContext.java | 1 -
.../pulsar/testutils/function/ControlSource.java | 228 ++++++++++++
.../pulsar/testutils/runtime/PulsarRuntime.java | 36 +-
.../testutils/runtime/PulsarRuntimeOperator.java | 414 ++++++++++++++++++---
.../runtime/container/PulsarContainerRuntime.java | 61 ++-
.../runtime/embedded/PulsarEmbeddedRuntime.java | 284 ++++++++++++++
.../runtime/mock/BlankBrokerInterceptor.java | 61 ---
.../runtime/mock/MockBookKeeperClientFactory.java | 74 ----
.../testutils/runtime/mock/MockPulsarService.java | 87 -----
.../runtime/mock/MockZooKeeperClientFactory.java | 73 ----
.../runtime/mock/NonClosableMockBookKeeper.java | 55 ---
.../testutils/runtime/mock/PulsarMockRuntime.java | 160 --------
.../mock/SameThreadOrderedSafeExecutor.java | 56 ---
.../test/resources/containers/txnStandalone.conf | 100 ++++-
.../src/main/resources/META-INF/NOTICE | 16 +-
.../util/flink/container/FlinkContainers.java | 2 +-
.../flink-end-to-end-tests-pulsar/pom.xml | 43 ++-
.../util/pulsar/PulsarSourceOrderedE2ECase.java | 7 +-
.../util/pulsar/PulsarSourceUnorderedE2ECase.java | 7 +-
.../pulsar/cases/ExclusiveSubscriptionContext.java | 14 -
.../pulsar/cases/FailoverSubscriptionContext.java | 14 -
.../pulsar/cases/KeySharedSubscriptionContext.java | 7 +-
.../pulsar/cases/SharedSubscriptionContext.java | 7 +-
.../FlinkContainerWithPulsarEnvironment.java | 5 +
.../common/PulsarContainerTestEnvironment.java | 17 +-
.../pyflink/datastream/tests/test_connectors.py | 2 +-
.../org/apache/flink/util/DockerImageVersions.java | 2 +-
109 files changed, 6217 insertions(+), 1485 deletions(-)
rename
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/{PulsarConfigUtils.java
=> PulsarClientFactory.java} (55%)
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigBuilder.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigValidator.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfiguration.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSink.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSinkBuilder.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSinkOptions.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommittable.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommittableSerializer.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommitter.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/config/PulsarSinkConfigUtils.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/config/SinkConfiguration.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/PulsarWriter.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/context/PulsarSinkContext.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/context/PulsarSinkContextImpl.java
copy
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/{source/config/CursorVerification.java
=> sink/writer/delayer/FixedMessageDelayer.java} (53%)
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/delayer/MessageDelayer.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessage.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessageBuilder.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/KeyHashTopicRouter.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/MessageKeyHash.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/RoundRobinTopicRouter.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/TopicRouter.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/TopicRoutingMode.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSchemaWrapper.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSerializationSchema.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSerializationSchemaWrapper.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/topic/TopicMetadataListener.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/topic/TopicProducerRegister.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigBuilderTest.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigValidatorTest.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigurationTest.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/sink/PulsarSinkBuilderTest.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/sink/PulsarSinkITCase.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommittableSerializerTest.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/sink/writer/PulsarWriterTest.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/sink/writer/router/KeyHashTopicRouterTest.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/sink/writer/router/RoundRobinTopicRouterTest.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/sink/writer/topic/TopicMetadataListenerTest.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/sink/writer/topic/TopicProducerRegisterTest.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/function/ControlSource.java
create mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/embedded/PulsarEmbeddedRuntime.java
delete mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/mock/BlankBrokerInterceptor.java
delete mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/mock/MockBookKeeperClientFactory.java
delete mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/mock/MockPulsarService.java
delete mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/mock/MockZooKeeperClientFactory.java
delete mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/mock/NonClosableMockBookKeeper.java
delete mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/mock/PulsarMockRuntime.java
delete mode 100644
flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/mock/SameThreadOrderedSafeExecutor.java
copy
flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/config/CursorVerification.java
=>
flink-end-to-end-tests/flink-end-to-end-tests-pulsar/src/test/java/org/apache/flink/tests/util/pulsar/common/PulsarContainerTestEnvironment.java
(58%)
