This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
from c37b85b2f [Docs][Zeta] Add savepoint doc (#5081)
add d729fcba4 [Feature][connector-v2][mongodbcdc]Support source mongodb
cdc (#4923)
No new revisions were added by this update.
Summary of changes:
LICENSE | 3 +-
config/plugin_config | 1 +
docs/en/connector-v2/source/MongoDB-CDC.md | 282 ++++++++++++++
plugin-mapping.properties | 1 +
.../external/IncrementalSourceStreamFetcher.java | 14 +-
.../connector-cdc-mongodb}/pom.xml | 78 ++--
.../cdc/mongodb/MongodbIncrementalSource.java | 132 +++++++
.../mongodb/MongodbIncrementalSourceFactory.java} | 70 ++--
.../cdc/mongodb/config/MongodbSourceConfig.java | 122 ++++++
.../config/MongodbSourceConfigProvider.java | 175 +++++++++
.../cdc/mongodb/config/MongodbSourceOptions.java | 255 +++++++++++++
.../exception/MongodbConnectorException.java | 9 +-
.../mongodb/internal/MongodbClientProvider.java | 45 +++
.../MongoDBConnectorDeserializationSchema.java} | 163 +++++++--
.../cdc/mongodb/sender}/SerializableFunction.java | 2 +-
.../cdc/mongodb/source/dialect/MongodbDialect.java | 146 ++++++++
.../source/fetch/MongodbFetchTaskContext.java | 210 +++++++++++
.../mongodb/source/fetch/MongodbScanFetchTask.java | 253 +++++++++++++
.../source/fetch/MongodbStreamFetchTask.java | 359 ++++++++++++++++++
.../source/offset/ChangeStreamDescriptor.java | 73 ++++
.../mongodb/source/offset/ChangeStreamOffset.java | 94 +++++
.../source/offset/ChangeStreamOffsetFactory.java | 62 ++++
.../source/splitters/MongodbChunkSplitter.java} | 24 +-
.../splitters/SampleBucketSplitStrategy.java | 139 +++++++
.../source/splitters/ShardedSplitStrategy.java | 108 ++++++
.../source/splitters/SingleSplitStrategy.java | 54 +++
.../cdc/mongodb/source/splitters/SplitContext.java | 93 +++++
.../mongodb/source/splitters/SplitStrategy.java | 57 +++
.../source/splitters/SplitVectorSplitStrategy.java | 119 ++++++
.../seatunnel/cdc/mongodb/utils/BsonUtils.java | 325 ++++++++++++++++
.../seatunnel/cdc/mongodb/utils/ChunkUtils.java | 34 +-
.../mongodb/utils/CollectionDiscoveryUtils.java | 171 +++++++++
.../cdc/mongodb/utils/MongodbRecordUtils.java | 165 +++++++++
.../seatunnel/cdc/mongodb/utils/MongodbUtils.java | 407 +++++++++++++++++++++
.../seatunnel/cdc/mongodb/utils/ResumeToken.java | 80 ++++
.../MongodbIncrementalSourceFactoryTest.java} | 8 +-
.../cdc/mysql/testutils/UniqueDatabase.java | 10 +-
seatunnel-connectors-v2/connector-cdc/pom.xml | 1 +
seatunnel-dist/pom.xml | 6 +
.../pom.xml | 36 +-
.../src/test/java/mongodb/MongoDBContainer.java | 240 ++++++++++++
.../src/test/java/mongodb/MongodbCDCIT.java | 266 ++++++++++++++
.../src/test/resources/ddl/inventory.js | 24 ++
.../src/test/resources/ddl/inventoryDDL.js | 32 ++
.../src/test/resources/ddl/mongodb_cdc.sql} | 26 +-
.../src/test/resources/docker/mongodb/random.key | 34 ++
.../src/test/resources/docker/mongodb/setup.js | 39 ++
.../src/test/resources/log4j2-test.properties | 4 +-
.../src/test/resources/mongodbcdc_to_mysql.conf | 46 ++-
seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml | 1 +
50 files changed, 4891 insertions(+), 207 deletions(-)
create mode 100644 docs/en/connector-v2/source/MongoDB-CDC.md
copy seatunnel-connectors-v2/{connector-file/connector-file-s3 =>
connector-cdc/connector-cdc-mongodb}/pom.xml (52%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/MongodbIncrementalSource.java
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactory.java
=>
connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/MongodbIncrementalSourceFactory.java}
(54%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/config/MongodbSourceConfig.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/config/MongodbSourceConfigProvider.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/config/MongodbSourceOptions.java
copy
seatunnel-connectors-v2/{connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel
=>
connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc}/mongodb/exception/MongodbConnectorException.java
(78%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/internal/MongodbClientProvider.java
copy
seatunnel-connectors-v2/{connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/serde/BsonToRowDataConverters.java
=>
connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/sender/MongoDBConnectorDeserializationSchema.java}
(71%)
copy
seatunnel-connectors-v2/{connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/serde
=>
connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/sender}/SerializableFunction.java
(93%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/dialect/MongodbDialect.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/fetch/MongodbFetchTaskContext.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/fetch/MongodbScanFetchTask.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/fetch/MongodbStreamFetchTask.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/offset/ChangeStreamDescriptor.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/offset/ChangeStreamOffset.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/offset/ChangeStreamOffsetFactory.java
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/splitter/ChunkSplitter.java
=>
connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/splitters/MongodbChunkSplitter.java}
(50%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/splitters/SampleBucketSplitStrategy.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/splitters/ShardedSplitStrategy.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/splitters/SingleSplitStrategy.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/splitters/SplitContext.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/splitters/SplitStrategy.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/source/splitters/SplitVectorSplitStrategy.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/BsonUtils.java
copy
seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/exception/FieldMapperTransformErrorCode.java
=>
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/ChunkUtils.java
(54%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/CollectionDiscoveryUtils.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/MongodbRecordUtils.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/MongodbUtils.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mongodb/utils/ResumeToken.java
copy
seatunnel-connectors-v2/{connector-console/src/test/java/org/apache/seatunnel/connectors/seatunnel/console/ConsoleFactoryTest.java
=>
connector-cdc/connector-cdc-mongodb/src/test/java/mongodb/source/MongodbIncrementalSourceFactoryTest.java}
(78%)
copy seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e =>
connector-cdc-mongodb-e2e}/pom.xml (74%)
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/src/test/java/mongodb/MongoDBContainer.java
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/src/test/java/mongodb/MongodbCDCIT.java
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/src/test/resources/ddl/inventory.js
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/src/test/resources/ddl/inventoryDDL.js
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e/src/test/resources/docker/setup.sql
=> connector-cdc-mongodb-e2e/src/test/resources/ddl/mongodb_cdc.sql} (59%)
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/src/test/resources/docker/mongodb/random.key
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/src/test/resources/docker/mongodb/setup.js
copy seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e =>
connector-cdc-mongodb-e2e}/src/test/resources/log4j2-test.properties (91%)
copy
seatunnel-engine/seatunnel-engine-server/src/test/resources/fake_to_console.conf
=>
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/src/test/resources/mongodbcdc_to_mysql.conf
(58%)
