This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
from 92f847c99f [Improve][Connector-V2] oracle cdc e2e optimization (#6232)
add 18d3e86194 [Feature] Supports iceberg sink #6198 (#6265)
No new revisions were added by this update.
Summary of changes:
docs/en/connector-v2/sink/Iceberg.md | 182 ++++++++
docs/en/connector-v2/source/Iceberg.md | 40 +-
plugin-mapping.properties | 1 +
pom.xml | 3 +-
release-note.md | 1 +
.../connector-cdc/connector-cdc-base/pom.xml | 14 +
seatunnel-connectors-v2/connector-iceberg/pom.xml | 72 ++-
.../seatunnel/iceberg/IcebergCatalogFactory.java | 88 ----
.../seatunnel/iceberg/IcebergCatalogLoader.java | 124 ++++++
.../seatunnel/iceberg/IcebergTableLoader.java | 66 ++-
.../seatunnel/iceberg/catalog/IcebergCatalog.java | 263 +++++++++++
.../iceberg/catalog/IcebergCatalogFactory.java | 20 +-
.../seatunnel/iceberg/config/CommonConfig.java | 87 ++--
.../seatunnel/iceberg/config/SinkConfig.java | 145 ++++++
.../seatunnel/iceberg/config/SourceConfig.java | 8 +-
.../seatunnel/iceberg/data/IcebergTypeMapper.java | 57 +++
.../seatunnel/iceberg/data/RowConverter.java | 495 +++++++++++++++++++++
.../seatunnel/iceberg/sink/IcebergSink.java | 214 +++++++++
.../seatunnel/iceberg/sink/IcebergSinkFactory.java | 112 +++++
.../seatunnel/iceberg/sink/IcebergSinkWriter.java | 160 +++++++
.../sink/commit/IcebergAggregatedCommitInfo.java} | 6 +-
.../sink/commit/IcebergAggregatedCommitter.java | 71 +++
.../iceberg/sink/commit/IcebergCommitInfo.java} | 8 +-
.../iceberg/sink/commit/IcebergFilesCommitter.java | 91 ++++
.../iceberg/sink/schema/ISchemaChange.java | 7 +-
.../iceberg/sink/schema/SchemaAddColumn.java | 35 +-
.../iceberg/sink/schema/SchemaChangeColumn.java} | 24 +-
.../iceberg/sink/schema/SchemaChangeWrapper.java | 72 +++
.../iceberg/sink/schema/SchemaDeleteColumn.java | 19 +-
.../iceberg/sink/schema/SchemaModifyColumn.java | 22 +-
.../iceberg/sink/state/IcebergSinkState.java | 23 +-
.../iceberg/sink/writer/BaseDeltaTaskWriter.java | 126 ++++++
.../iceberg/sink/writer/IcebergRecord.java | 87 ++++
.../iceberg/sink/writer/IcebergRecordWriter.java | 175 ++++++++
.../iceberg/sink/writer/IcebergWriterFactory.java | 197 ++++++++
.../sink/writer/PartitionedAppendWriter.java | 56 +++
.../sink/writer/PartitionedDeltaWriter.java | 95 ++++
.../iceberg/sink/writer/RecordProjection.java | 196 ++++++++
.../iceberg/sink/writer/RecordWriter.java | 24 +-
.../sink/writer/UnpartitionedDeltaWriter.java | 68 +++
.../seatunnel/iceberg/sink/writer/WriteResult.java | 29 +-
.../seatunnel/iceberg/source/IcebergSource.java | 55 ++-
.../iceberg/source/IcebergSourceFactory.java | 56 ++-
.../source/enumerator/AbstractSplitEnumerator.java | 8 +-
.../enumerator/IcebergBatchSplitEnumerator.java | 7 +-
.../enumerator/IcebergStreamSplitEnumerator.java | 7 +-
.../source/reader/IcebergFileScanTaskReader.java | 6 +-
.../iceberg/source/reader/IcebergSourceReader.java | 9 +-
.../seatunnel/iceberg/utils/SchemaUtils.java | 322 ++++++++++++++
.../seatunnel/iceberg/TestIcebergMetastore.java | 20 +-
.../iceberg/catalog/IcebergCatalogTest.java | 172 +++++++
.../connector-iceberg-e2e/pom.xml | 54 +++
.../e2e/connector/iceberg/IcebergSinkCDCIT.java | 459 +++++++++++++++++++
.../e2e/connector/iceberg/IcebergSinkIT.java | 169 +++++++
.../e2e/connector/iceberg/IcebergSourceIT.java | 21 +-
.../src/test/resources/ddl/inventory.sql | 0
.../src/test/resources/ddl/mysql_cdc.sql | 186 ++++++++
.../test/resources/iceberg/fake_to_iceberg.conf} | 23 +-
.../src/test/resources/iceberg/iceberg_source.conf | 6 +-
.../resources/iceberg/mysql_cdc_to_iceberg.conf} | 52 ++-
.../mysql_cdc_to_iceberg_for_schema_change.conf} | 56 ++-
.../src/test/resources/mysql}/server-gtids/my.cnf | 0
.../src/test/resources/mysql}/setup.sql | 4 +-
.../connector/iceberg/hadoop3/IcebergSourceIT.java | 21 +-
.../src/test/resources/iceberg/iceberg_source.conf | 6 +-
.../seatunnel-hadoop3-3.1.4-uber/pom.xml | 1 -
66 files changed, 4913 insertions(+), 390 deletions(-)
create mode 100644 docs/en/connector-v2/sink/Iceberg.md
delete mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/IcebergCatalogFactory.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/IcebergCatalogLoader.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/catalog/IcebergCatalog.java
copy
seatunnel-api/src/test/java/org/apache/seatunnel/api/table/catalog/InMemoryCatalogFactory.java
=>
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/catalog/IcebergCatalogFactory.java
(64%)
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/config/SinkConfig.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/RowConverter.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/IcebergSink.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/IcebergSinkFactory.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/IcebergSinkWriter.java
copy
seatunnel-connectors-v2/{connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/sink/state/MongodbAggregatedCommitInfo.java
=>
connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/commit/IcebergAggregatedCommitInfo.java}
(84%)
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/commit/IcebergAggregatedCommitter.java
copy
seatunnel-connectors-v2/{connector-kafka/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/state/KafkaAggregatedCommitInfo.java
=>
connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/commit/IcebergCommitInfo.java}
(79%)
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/commit/IcebergFilesCommitter.java
copy
seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/runtime/ExecutionMode.java
=>
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/schema/ISchemaChange.java
(88%)
copy
seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/dag/internal/IntermediateQueue.java
=>
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/schema/SchemaAddColumn.java
(60%)
copy
seatunnel-connectors-v2/{connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/SingleSplit.java
=>
connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/schema/SchemaChangeColumn.java}
(64%)
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/schema/SchemaChangeWrapper.java
copy
seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/EngineType.java
=>
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/schema/SchemaDeleteColumn.java
(76%)
copy
seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/enums/PluginType.java
=>
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/schema/SchemaModifyColumn.java
(65%)
copy
seatunnel-engine/seatunnel-engine-storage/checkpoint-storage-api/src/main/java/org/apache/seatunnel/engine/checkpoint/storage/PipelineState.java
=>
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/state/IcebergSinkState.java
(66%)
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/writer/BaseDeltaTaskWriter.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/writer/IcebergRecord.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/writer/IcebergRecordWriter.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/writer/IcebergWriterFactory.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/writer/PartitionedAppendWriter.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/writer/PartitionedDeltaWriter.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/writer/RecordProjection.java
copy
seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/src/main/java/org/apache/seatunnel/engine/imap/storage/file/wal/reader/IFileReader.java
=>
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/writer/RecordWriter.java
(56%)
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/writer/UnpartitionedDeltaWriter.java
copy
seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/src/main/java/org/apache/seatunnel/engine/imap/storage/file/disruptor/FileWALEvent.java
=>
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/sink/writer/WriteResult.java
(68%)
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/utils/SchemaUtils.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/test/java/org/apache/seatunnel/connectors/seatunnel/iceberg/catalog/IcebergCatalogTest.java
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/IcebergSinkCDCIT.java
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/IcebergSinkIT.java
copy seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e =>
connector-iceberg-e2e}/src/test/resources/ddl/inventory.sql (100%)
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/ddl/mysql_cdc.sql
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-paimon-e2e/src/test/resources/fake_to_paimon.conf
=> connector-iceberg-e2e/src/test/resources/iceberg/fake_to_iceberg.conf} (77%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_disable_exactly_once.conf
=> connector-iceberg-e2e/src/test/resources/iceberg/mysql_cdc_to_iceberg.conf}
(62%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_disable_exactly_once.conf
=>
connector-iceberg-e2e/src/test/resources/iceberg/mysql_cdc_to_iceberg_for_schema_change.conf}
(60%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e/src/test/resources/docker
=> connector-iceberg-e2e/src/test/resources/mysql}/server-gtids/my.cnf (100%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-kafka-e2e/src/test/resources/docker
=> connector-iceberg-e2e/src/test/resources/mysql}/setup.sql (92%)
