This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
from e5e0cba26 [Hotfix][connector-clickhouse] fix get clickhouse local
table name with closing bracket from distributed table engineFull (#4710)
add 5b1a843e4 [Feature][Connector-v2][Mongodb]Refactor mongodb connector
(#4620)
No new revisions were added by this update.
Summary of changes:
docs/en/connector-v2/sink/MongoDB.md | 229 +++++++++--
docs/en/connector-v2/source/MongoDB.md | 453 ++++++++++++++++++---
seatunnel-connectors-v2/connector-mongodb/pom.xml | 18 +-
.../seatunnel/mongodb/config/MongodbConfig.java | 169 ++++++--
.../seatunnel/mongodb/config/MongodbOption.java | 51 ---
.../seatunnel/mongodb/data/DataTypeValidator.java | 53 ---
.../mongodb/data/DefaultDeserializer.java | 56 ---
.../seatunnel/mongodb/data/DefaultSerializer.java | 124 ------
.../seatunnel/mongodb/data/Deserializer.java | 28 --
.../mongodb/data/SeaTunnelRowBsonWriter.java | 332 ---------------
.../exception/MongodbConnectorException.java | 4 -
.../mongodb/internal/MongodbClientProvider.java} | 42 +-
.../internal/MongodbCollectionProvider.java | 59 +++
.../internal/MongodbSingleCollectionProvider.java | 97 +++++
.../mongodb/serde/BsonToRowDataConverters.java | 436 ++++++++++++++++++++
.../mongodb/serde/DocumentDeserializer.java} | 9 +-
.../mongodb/serde/DocumentRowDataDeserializer.java | 75 ++++
.../DocumentSerializer.java} | 11 +-
.../mongodb/serde/RowDataDocumentSerializer.java | 74 ++++
.../mongodb/serde/RowDataToBsonConverters.java | 332 +++++++++++++++
.../mongodb/serde/SerializableFunction.java} | 6 +-
...godbSinkFactory.java => MongoKeyExtractor.java} | 32 +-
.../seatunnel/mongodb/sink/MongodbSink.java | 85 ++--
.../seatunnel/mongodb/sink/MongodbSinkFactory.java | 23 +-
.../seatunnel/mongodb/sink/MongodbSinkWriter.java | 80 ----
.../seatunnel/mongodb/sink/MongodbWriter.java | 167 ++++++++
.../mongodb/sink/MongodbWriterOptions.java | 150 +++++++
.../seatunnel/mongodb/source/MongodbSource.java | 142 +++++--
.../mongodb/source/MongodbSourceFactory.java | 30 +-
.../mongodb/source/MongodbSourceReader.java | 118 ------
.../mongodb/source/config/MongodbReadOptions.java | 84 ++++
.../source/enumerator/MongodbSplitEnumerator.java | 167 ++++++++
.../mongodb/source/reader/MongodbReader.java | 145 +++++++
.../mongodb/source/split/MongoSplit.java} | 20 +-
.../mongodb/source/split/MongoSplitStrategy.java} | 12 +-
.../mongodb/source/split/MongoSplitUtils.java | 72 ++++
.../source/split/SamplingSplitStrategy.java | 252 ++++++++++++
.../e2e/connector/v2/mongodb/MongodbIT.java | 311 +++++++++-----
.../test/resources/fake_source_to_mongodb.conf} | 67 +--
.../flatIT/fake_source_to_flat_mongodb.conf} | 66 +--
.../flatIT/mongodb_flat_source_to_assert.conf} | 50 ++-
.../mongodb_matchProjection_source_to_assert.conf} | 75 ++--
.../mongodb_matchQuery_source_to_assert.conf} | 74 ++--
.../test/resources/mongodb_source_and_sink.conf | 70 ----
.../mongodb_source_matchQuery_and_sink.conf | 71 ----
.../test/resources/mongodb_source_to_assert.conf | 102 +++--
.../mongodb_split_key_source_to_assert.conf} | 74 ++--
.../mongodb_split_size_source_to_assert.conf} | 76 ++--
.../fake_source_to_updateMode_insert_mongodb.conf} | 69 ++--
.../updateIT/fake_source_to_update_mongodb.conf} | 69 ++--
.../updateIT/update_mongodb_to_assert.conf} | 37 +-
51 files changed, 3806 insertions(+), 1642 deletions(-)
delete mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/config/MongodbOption.java
delete mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/data/DataTypeValidator.java
delete mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/data/DefaultDeserializer.java
delete mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/data/DefaultSerializer.java
delete mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/data/Deserializer.java
delete mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/data/SeaTunnelRowBsonWriter.java
copy
seatunnel-connectors-v2/{connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/commit/FileCommitInfo.java
=>
connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/internal/MongodbClientProvider.java}
(51%)
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/internal/MongodbCollectionProvider.java
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/internal/MongodbSingleCollectionProvider.java
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/serde/BsonToRowDataConverters.java
copy
seatunnel-connectors-v2/{connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/SingleSplitEnumeratorState.java
=>
connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/serde/DocumentDeserializer.java}
(80%)
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/serde/DocumentRowDataDeserializer.java
rename
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/{data/Serializer.java
=> serde/DocumentSerializer.java} (76%)
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/serde/RowDataDocumentSerializer.java
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/serde/RowDataToBsonConverters.java
copy
seatunnel-connectors-v2/{connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/SingleSplitEnumeratorState.java
=>
connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/serde/SerializableFunction.java}
(80%)
copy
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/sink/{MongodbSinkFactory.java
=> MongoKeyExtractor.java} (53%)
delete mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/sink/MongodbSinkWriter.java
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/sink/MongodbWriter.java
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/sink/MongodbWriterOptions.java
delete mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/MongodbSourceReader.java
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/config/MongodbReadOptions.java
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/enumerator/MongodbSplitEnumerator.java
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/reader/MongodbReader.java
copy
seatunnel-connectors-v2/{connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/source/StarRocksSourceSplit.java
=>
connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/split/MongoSplit.java}
(76%)
copy
seatunnel-connectors-v2/{connector-elasticsearch/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/dto/source/ScrollResult.java
=>
connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/split/MongoSplitStrategy.java}
(78%)
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/split/MongoSplitUtils.java
create mode 100644
seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/split/SamplingSplitStrategy.java
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-file-sftp-e2e/src/test/resources/excel/fakesource_to_sftp_excel.conf
=> connector-mongodb-e2e/src/test/resources/fake_source_to_mongodb.conf} (61%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-file-sftp-e2e/src/test/resources/excel/fakesource_to_sftp_excel.conf
=>
connector-mongodb-e2e/src/test/resources/flatIT/fake_source_to_flat_mongodb.conf}
(61%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-file-local-e2e/src/test/resources/orc/local_file_orc_to_assert.conf
=>
connector-mongodb-e2e/src/test/resources/flatIT/mongodb_flat_source_to_assert.conf}
(70%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-file-sftp-e2e/src/test/resources/json/fake_to_sftp_file_json.conf
=>
connector-mongodb-e2e/src/test/resources/matchIT/mongodb_matchProjection_source_to_assert.conf}
(59%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-file-sftp-e2e/src/test/resources/json/fake_to_sftp_file_json.conf
=>
connector-mongodb-e2e/src/test/resources/matchIT/mongodb_matchQuery_source_to_assert.conf}
(59%)
delete mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-mongodb-e2e/src/test/resources/mongodb_source_and_sink.conf
delete mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-mongodb-e2e/src/test/resources/mongodb_source_matchQuery_and_sink.conf
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-file-sftp-e2e/src/test/resources/json/fake_to_sftp_file_json.conf
=>
connector-mongodb-e2e/src/test/resources/splitIT/mongodb_split_key_source_to_assert.conf}
(59%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-file-sftp-e2e/src/test/resources/json/fake_to_sftp_file_json.conf
=>
connector-mongodb-e2e/src/test/resources/splitIT/mongodb_split_size_source_to_assert.conf}
(58%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-file-sftp-e2e/src/test/resources/excel/fakesource_to_sftp_excel.conf
=>
connector-mongodb-e2e/src/test/resources/updateIT/fake_source_to_updateMode_insert_mongodb.conf}
(59%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-file-sftp-e2e/src/test/resources/excel/fakesource_to_sftp_excel.conf
=>
connector-mongodb-e2e/src/test/resources/updateIT/fake_source_to_update_mongodb.conf}
(59%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-fake-e2e/src/test/resources/fake_to_assert.conf
=>
connector-mongodb-e2e/src/test/resources/updateIT/update_mongodb_to_assert.conf}
(81%)
