This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12776-ac3316d83631cbd40660602afbbe7c87fec5d4fa in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit fae9ffa96cc9d0e3e0da90c3909b0ee173fd4ddd Author: Goutam Adwant <[email protected]> AuthorDate: Sat Nov 7 03:11:57 2026 +0000 [Fix][Translation] Support CDC sources on the Spark engine (#12776) --- docs/en/connectors/source/MySQL-CDC.md | 14 +- docs/en/connectors/source/PostgreSQL-CDC.md | 14 + docs/zh/connectors/source/MySQL-CDC.md | 12 +- docs/zh/connectors/source/PostgreSQL-CDC.md | 9 + .../connector-cdc/connector-cdc-base/pom.xml | 7 + .../cdc/base/source/IncrementalSource.java | 11 +- .../IncrementalSourceEnumeratorTest.java | 318 ++++++++++++ .../cdc/mysql/AbstractMysqlCDCITBase.java | 113 +++++ .../connectors/seatunnel/cdc/mysql/MysqlCDCIT.java | 28 +- .../test/resources/mysqlcdc_to_mysql_on_spark.conf | 69 +++ ...ysqlcdc_to_mysql_with_parallelism_on_spark.conf | 71 +++ .../source/CoordinatedEnumeratorContext.java | 4 +- .../translation/source/CoordinatedSource.java | 110 +++- .../translation/source/CoordinatedSourceTest.java | 559 +++++++++++++++++++++ .../spark/source/SeaTunnelSourceSupport.java | 11 + .../batch/CoordinatedBatchPartitionReader.java | 60 +-- .../batch/CoordinatedBatchPartitionReader.java | 60 +-- .../spark/source/scan/SeaTunnelScan.java | 11 + .../batch/CoordinatedBatchPartitionReaderTest.java | 433 ++++++++++++++++ 19 files changed, 1814 insertions(+), 100 deletions(-) diff --git a/docs/en/connectors/source/MySQL-CDC.md b/docs/en/connectors/source/MySQL-CDC.md index 5e210a71cc3..1df8363e7fc 100644 --- a/docs/en/connectors/source/MySQL-CDC.md +++ b/docs/en/connectors/source/MySQL-CDC.md @@ -6,6 +6,7 @@ import ChangeLog from '../changelog/connector-cdc-mysql.md'; ## Support Those Engines +> Spark<br/> > SeaTunnel Zeta<br/> > Flink <br/> @@ -33,7 +34,7 @@ describes how to set up the MySQL CDC connector to run SQL queries against MySQL ### Install Jdbc Driver -#### For Flink Engine +#### For Spark/Flink Engine > 1. You need to ensure that the [jdbc driver jar > package](https://mvnrepository.com/artifact/mysql/mysql-connector-java) has > been placed in directory `${SEATUNNEL_HOME}/plugins/`. @@ -689,6 +690,17 @@ latest offset and skip the initial snapshot. Set `server-time-zone` to match the MySQL server's timezone, for example `"Asia/Shanghai"`. For character set issues, append `characterEncoding=UTF-8&useUnicode=true` to the JDBC connection URL. +### Can MySQL CDC run on the Spark engine? + +Yes. The Spark engine runs the source as one Spark task that holds the split enumerator and all +readers, for both `BATCH` and `STREAMING` jobs; `parallelism` sets the number of readers in that task. +Spark does not checkpoint the source, so: + +- A job cannot be restored from a checkpoint. A restarted job starts again from `startup.mode`. +- A sink that only writes on checkpoints or full batches can hold rows back, for example the JDBC + sink with its default `batch_size`. Use a smaller `batch_size` when rows must arrive promptly. +- Reading the source with Spark micro-batch streaming (`readStream`) is not supported. + ## See Also For a production-grade end-to-end guide covering full + incremental synchronization lifecycle, diff --git a/docs/en/connectors/source/PostgreSQL-CDC.md b/docs/en/connectors/source/PostgreSQL-CDC.md index 9eb0ad5d37a..4215ebbbe31 100644 --- a/docs/en/connectors/source/PostgreSQL-CDC.md +++ b/docs/en/connectors/source/PostgreSQL-CDC.md @@ -424,6 +424,20 @@ may replay a row at an ambiguous boundary to avoid losing it. Replication lag can occur when the logical decoding plugin is slow or when the WAL sender is under load. Monitor `pg_replication_slots` for `confirmed_flush_lsn` drift. Ensure the CDC job consumes events continuously and that network latency between SeaTunnel and PostgreSQL is low. +### Can PostgreSQL CDC run on the Spark engine? + +It can, but it is not covered by end-to-end tests. The Spark engine runs CDC sources as one Spark +task that holds the split enumerator and all readers, for both `BATCH` and `STREAMING` jobs. Spark +does not checkpoint the source, so: + +- A job cannot be restored from a checkpoint. A restarted job starts again from `startup.mode`. +- SeaTunnel never confirms a processed LSN to the replication slot, so the slot's `restart_lsn` does + not move and PostgreSQL keeps all WAL from it for as long as the slot exists. Watch the WAL size, + and advance or drop the slot when the job stops. +- A sink that only writes on checkpoints or full batches can hold rows back, for example the JDBC + sink with its default `batch_size`. +- Reading the source with Spark micro-batch streaming (`readStream`) is not supported. + ## See Also For a production-grade end-to-end guide covering full + incremental synchronization lifecycle, diff --git a/docs/zh/connectors/source/MySQL-CDC.md b/docs/zh/connectors/source/MySQL-CDC.md index 62748f4ccc8..4c5354c5a4c 100644 --- a/docs/zh/connectors/source/MySQL-CDC.md +++ b/docs/zh/connectors/source/MySQL-CDC.md @@ -6,6 +6,7 @@ import ChangeLog from '../changelog/connector-cdc-mysql.md'; ## 支持这些引擎 +> Spark<br/> > SeaTunnel Zeta<br/> > Flink <br/> @@ -32,7 +33,7 @@ MySQL CDC连接器允许从MySQL数据库读取快照和增量数据. 本文档 ### 安装Jdbc驱动 -#### 对于Flink引擎 +#### 对于Spark/Flink引擎 > 1. 你需要确保 [jdbc 驱动 jar > package](https://mvnrepository.com/artifact/mysql/mysql-connector-java) > 已经放在目录 `${SEATUNNEL_HOME}/plugins/`. @@ -676,6 +677,15 @@ source options 示例那样,通过 `table-names-config.primaryKeys` 指定自 将 `server-time-zone` 设置为与 MySQL 服务器一致的时区,例如 `"Asia/Shanghai"`。字符集问题可通过在 JDBC 连接 URL 中追加 `characterEncoding=UTF-8&useUnicode=true` 来解决。 +### MySQL CDC 能在 Spark 引擎上运行吗? + +可以。无论 `BATCH` 还是 `STREAMING` 作业,Spark 引擎都在一个 Spark task 中运行该数据源,这个 task 同时包含分片枚举器和所有 reader,`parallelism` 决定其中 reader 的数量。 +Spark 不会为该数据源做 checkpoint,因此: + +- 作业无法从 checkpoint 恢复,重启的作业会按 `startup.mode` 重新开始。 +- 只在 checkpoint 或批次写满时才写出的 sink 可能会暂时攒住数据,例如使用默认 `batch_size` 的 JDBC sink。需要数据及时写出时,请调小 `batch_size`。 +- 不支持通过 Spark 微批流式读取(`readStream`)该数据源。 + ## 另请参阅 若需要一份面向生产的端到端实践指南,涵盖全量 + 增量同步生命周期、2PC sink 配置、Schema 演进与常见故障排查,请参阅 [CDC 生产实战手册](../cdc-production-cookbook.md)。 diff --git a/docs/zh/connectors/source/PostgreSQL-CDC.md b/docs/zh/connectors/source/PostgreSQL-CDC.md index 078cc2a99d7..6c9d700f3a6 100644 --- a/docs/zh/connectors/source/PostgreSQL-CDC.md +++ b/docs/zh/connectors/source/PostgreSQL-CDC.md @@ -409,6 +409,15 @@ PostgreSQL 13 及以上版本会在某些情况下使复制槽失效,例如复 滞后可能由逻辑解码插件处理慢或 WAL sender 负载过高引起。可通过监控 `pg_replication_slots` 中的 `confirmed_flush_lsn` 漂移情况来排查。确保 CDC 任务持续消费事件,并保持 SeaTunnel 与 PostgreSQL 之间的网络低延迟。 +### PostgreSQL CDC 能在 Spark 引擎上运行吗? + +可以,但这一用法没有端到端测试覆盖。无论 `BATCH` 还是 `STREAMING` 作业,Spark 引擎都在一个 Spark task 中运行 CDC 数据源,这个 task 同时包含分片枚举器和所有 reader。Spark 不会为该数据源做 checkpoint,因此: + +- 作业无法从 checkpoint 恢复,重启的作业会按 `startup.mode` 重新开始。 +- SeaTunnel 不会向复制槽确认已处理的 LSN,因此复制槽的 `restart_lsn` 不会前移,只要复制槽存在,PostgreSQL 就会保留从该位置起的全部 WAL。请关注 WAL 占用,并在作业停止后推进或删除该复制槽。 +- 只在 checkpoint 或批次写满时才写出的 sink 可能会暂时攒住数据,例如使用默认 `batch_size` 的 JDBC sink。 +- 不支持通过 Spark 微批流式读取(`readStream`)该数据源。 + ## 另请参阅 若需要一份面向生产的端到端实践指南,涵盖全量 + 增量同步生命周期、2PC sink 配置、Schema 演进与常见故障排查,请参阅 [CDC 生产实战手册](../cdc-production-cookbook.md)。 diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/pom.xml b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/pom.xml index 5666992d526..0d21c3b2d8e 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/pom.xml +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/pom.xml @@ -118,6 +118,13 @@ <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.seatunnel</groupId> + <artifactId>seatunnel-translation-base</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + </dependencies> <build> diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java index 1e263508e71..2b97a41ed94 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java @@ -25,6 +25,7 @@ import org.apache.seatunnel.api.source.Boundedness; import org.apache.seatunnel.api.source.SeaTunnelSource; import org.apache.seatunnel.api.source.SourceReader; import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.source.SupportCoordinate; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; import org.apache.seatunnel.api.table.catalog.Column; @@ -85,10 +86,18 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; +/** + * Base class of the incremental snapshot CDC sources. + * + * <p>Its readers and enumerator exchange {@link org.apache.seatunnel.api.source.SourceEvent}s + * (finished snapshot split reports, their acknowledgements and the end of the snapshot phase), so + * it is marked with {@link SupportCoordinate} to run one enumerator for all readers on engines that + * would otherwise give every reader its own enumerator. + */ @NoArgsConstructor @Slf4j public abstract class IncrementalSource<T, C extends SourceConfig> - implements SeaTunnelSource<T, SourceSplitBase, PendingSplitsState> { + implements SeaTunnelSource<T, SourceSplitBase, PendingSplitsState>, SupportCoordinate { static { // Load DriverManager first to avoid deadlock between DriverManager's diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/IncrementalSourceEnumeratorTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/IncrementalSourceEnumeratorTest.java index 86c5e7269a7..5d46a244c04 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/IncrementalSourceEnumeratorTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/IncrementalSourceEnumeratorTest.java @@ -19,25 +19,55 @@ package org.apache.seatunnel.connectors.cdc.base.source.enumerator; import org.apache.seatunnel.api.common.metrics.MetricsContext; import org.apache.seatunnel.api.event.EventListener; +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SeaTunnelSource; import org.apache.seatunnel.api.source.SourceEvent; +import org.apache.seatunnel.api.source.SourceReader; import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.source.SupportCoordinate; +import org.apache.seatunnel.connectors.cdc.base.config.SourceConfig; +import org.apache.seatunnel.connectors.cdc.base.config.StartupConfig; +import org.apache.seatunnel.connectors.cdc.base.config.StopConfig; +import org.apache.seatunnel.connectors.cdc.base.dialect.DataSourceDialect; +import org.apache.seatunnel.connectors.cdc.base.option.StopMode; +import org.apache.seatunnel.connectors.cdc.base.source.enumerator.splitter.ChunkSplitter; +import org.apache.seatunnel.connectors.cdc.base.source.enumerator.state.HybridPendingSplitsState; import org.apache.seatunnel.connectors.cdc.base.source.enumerator.state.PendingSplitsState; +import org.apache.seatunnel.connectors.cdc.base.source.enumerator.state.SnapshotPhaseState; +import org.apache.seatunnel.connectors.cdc.base.source.event.CompletedSnapshotSplitsReportEvent; import org.apache.seatunnel.connectors.cdc.base.source.event.SnapshotSplitWatermark; +import org.apache.seatunnel.connectors.cdc.base.source.offset.Offset; +import org.apache.seatunnel.connectors.cdc.base.source.offset.OffsetFactory; +import org.apache.seatunnel.connectors.cdc.base.source.split.IncrementalSplit; import org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit; import org.apache.seatunnel.connectors.cdc.base.source.split.SourceSplitBase; +import org.apache.seatunnel.translation.source.CoordinatedSource; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import io.debezium.relational.TableId; + +import java.lang.reflect.Proxy; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Queue; import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Collectors; +import java.util.stream.IntStream; public class IncrementalSourceEnumeratorTest { + private static final TableId TABLE_ID = TableId.parse("db1.table1"); + @Test public void shouldAssignRestoredSplitsToWaitingReader() throws Exception { TestingEnumeratorContext context = new TestingEnumeratorContext(); @@ -81,6 +111,42 @@ public class IncrementalSourceEnumeratorTest { Collections.singletonList(restoredSplit), context.allAssignedSplits); } + /** + * Runs the real enumerator and hybrid split assigner through the coordinated source that the + * Spark engine uses, where all readers share one enumerator and no engine checkpoint is ever + * taken. + */ + @Test + void shouldEnterIncrementalPhaseWithMultipleReadersAndEnumeratorCheckpoints() throws Exception { + int parallelism = 2; + List<String> snapshotSplitIds = + IntStream.range(0, 4) + .mapToObj(i -> TABLE_ID + ":" + i) + .collect(Collectors.toList()); + ReportingSource source = new ReportingSource(snapshotSplitIds); + CoordinatedSource<String, SourceSplitBase, PendingSplitsState> coordinatedSource = + new CoordinatedSource<String, SourceSplitBase, PendingSplitsState>( + source, null, parallelism, "incremental-source-enumerator") { + @Override + protected long getEnumeratorCheckpointInterval() { + return 10L; + } + }; + + Assertions.assertEquals(parallelism, source.enumeratorParallelism); + + RecordCollector collector = new RecordCollector(); + coordinatedSource.open(); + Assertions.assertTimeoutPreemptively( + Duration.ofSeconds(30), () -> coordinatedSource.run(collector)); + coordinatedSource.close(); + + List<String> expected = new ArrayList<>(snapshotSplitIds); + expected.add("incremental-split-0"); + Assertions.assertEquals( + expected, collector.records.stream().sorted().collect(Collectors.toList())); + } + private static final class TestingSplitAssigner implements SplitAssigner { private List<SourceSplitBase> addedSplits = Collections.emptyList(); private List<SourceSplitBase> restoredSplits = Collections.emptyList(); @@ -159,4 +225,256 @@ public class IncrementalSourceEnumeratorTest { return null; } } + + private static final class RecordCollector implements Collector<String> { + + private final Object lock = new Object(); + private final Queue<String> records = new ConcurrentLinkedQueue<>(); + + @Override + public void collect(String record) { + records.add(record); + } + + @Override + public Object getCheckpointLock() { + return lock; + } + } + + /** Builds the real enumerator over a fixed set of snapshot splits of one table. */ + private static final class ReportingSource + implements SeaTunnelSource<String, SourceSplitBase, PendingSplitsState>, + SupportCoordinate { + + private final List<String> snapshotSplitIds; + private int enumeratorParallelism = -1; + + private ReportingSource(List<String> snapshotSplitIds) { + this.snapshotSplitIds = snapshotSplitIds; + } + + @Override + public Boundedness getBoundedness() { + return Boundedness.BOUNDED; + } + + @Override + public SourceReader<String, SourceSplitBase> createReader( + SourceReader.Context readerContext) { + return new ReportingReader(readerContext); + } + + @Override + public SourceSplitEnumerator<SourceSplitBase, PendingSplitsState> createEnumerator( + SourceSplitEnumerator.Context<SourceSplitBase> enumeratorContext) { + enumeratorParallelism = enumeratorContext.currentParallelism(); + List<SnapshotSplit> remainingSplits = + snapshotSplitIds.stream() + .map(id -> new SnapshotSplit(id, TABLE_ID, null, null, null)) + .collect(Collectors.toList()); + SnapshotPhaseState snapshotPhaseState = + new SnapshotPhaseState( + Collections.singletonList(TABLE_ID), + remainingSplits, + new HashMap<>(), + new HashMap<>(), + false, + Collections.emptyList(), + false, + true); + SplitAssigner.Context<SourceConfig> assignerContext = + new SplitAssigner.Context<>( + new TestSourceConfig(), + Collections.singleton(TABLE_ID), + new HashMap<>(), + new HashMap<>()); + HybridSplitAssigner<SourceConfig> splitAssigner = + new HybridSplitAssigner<>( + assignerContext, + enumeratorParallelism, + 1, + new HybridPendingSplitsState(snapshotPhaseState, null), + testDialect(), + new TestOffsetFactory()); + return new IncrementalSourceEnumerator(enumeratorContext, splitAssigner); + } + + @Override + public SourceSplitEnumerator<SourceSplitBase, PendingSplitsState> restoreEnumerator( + SourceSplitEnumerator.Context<SourceSplitBase> enumeratorContext, + PendingSplitsState checkpointState) { + throw new UnsupportedOperationException(); + } + + @Override + public String getPluginName() { + return "reporting-cdc-source"; + } + } + + /** + * Follows the protocol of the CDC reader: it reports every finished snapshot split to the + * enumerator and then asks for the next split. + */ + private static final class ReportingReader implements SourceReader<String, SourceSplitBase> { + + private final SourceReader.Context context; + private final Queue<SourceSplitBase> splits = new ConcurrentLinkedQueue<>(); + private volatile boolean started; + private volatile boolean noMoreSplits; + + private ReportingReader(SourceReader.Context context) { + this.context = context; + } + + @Override + public void open() {} + + @Override + public void close() {} + + @Override + public void pollNext(Collector<String> output) { + if (!started) { + started = true; + context.sendSplitRequest(); + return; + } + SourceSplitBase split = splits.poll(); + if (split == null) { + if (noMoreSplits) { + context.signalNoMoreElement(); + } + return; + } + output.collect(split.splitId()); + if (split.isSnapshotSplit()) { + CompletedSnapshotSplitsReportEvent reportEvent = + new CompletedSnapshotSplitsReportEvent(); + reportEvent.setCompletedSnapshotSplitWatermarks( + Collections.singletonList( + new SnapshotSplitWatermark( + split.splitId(), new TestOffset(1L), new TestOffset(2L)))); + context.sendSourceEventToEnumerator(reportEvent); + } else { + IncrementalSplit incrementalSplit = split.asIncrementalSplit(); + Assertions.assertEquals( + Collections.singletonList(TABLE_ID), incrementalSplit.getTableIds()); + } + context.sendSplitRequest(); + } + + @Override + public List<SourceSplitBase> snapshotState(long checkpointId) { + return new ArrayList<>(splits); + } + + @Override + public void addSplits(List<SourceSplitBase> splits) { + this.splits.addAll(splits); + } + + @Override + public void handleNoMoreSplits() { + noMoreSplits = true; + } + + @Override + public void notifyCheckpointComplete(long checkpointId) {} + } + + @SuppressWarnings("unchecked") + private static DataSourceDialect<SourceConfig> testDialect() { + ChunkSplitter chunkSplitter = + tableId -> { + throw new UnsupportedOperationException(); + }; + return (DataSourceDialect<SourceConfig>) + Proxy.newProxyInstance( + IncrementalSourceEnumeratorTest.class.getClassLoader(), + new Class<?>[] {DataSourceDialect.class}, + (proxy, method, args) -> { + if ("createChunkSplitter".equals(method.getName())) { + return chunkSplitter; + } + throw new UnsupportedOperationException(method.getName()); + }); + } + + private static final class TestSourceConfig implements SourceConfig { + + private static final long serialVersionUID = 1L; + + @Override + public StartupConfig getStartupConfig() { + return null; + } + + @Override + public StopConfig getStopConfig() { + return new StopConfig(StopMode.NEVER, null, null, null); + } + + @Override + public int getSplitSize() { + return 1; + } + + @Override + public boolean isExactlyOnce() { + return false; + } + } + + private static final class TestOffsetFactory extends OffsetFactory { + + private static final long serialVersionUID = 1L; + + @Override + public Offset earliest() { + throw new UnsupportedOperationException(); + } + + @Override + public Offset neverStop() { + return new TestOffset(Long.MAX_VALUE); + } + + @Override + public Offset latest() { + throw new UnsupportedOperationException(); + } + + @Override + public Offset specific(Map<String, String> offset) { + throw new UnsupportedOperationException(); + } + + @Override + public Offset specific(String filename, Long position) { + throw new UnsupportedOperationException(); + } + + @Override + public Offset timestamp(long timestamp) { + throw new UnsupportedOperationException(); + } + } + + private static final class TestOffset extends Offset { + + private static final long serialVersionUID = 1L; + + private TestOffset(long value) { + this.offset = Collections.singletonMap("pos", String.valueOf(value)); + } + + @Override + public int compareTo(Offset other) { + return Long.compare( + Long.parseLong(this.offset.get("pos")), + Long.parseLong(other.getOffset().get("pos"))); + } + } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/AbstractMysqlCDCITBase.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/AbstractMysqlCDCITBase.java index e403266b80a..b5acac6d394 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/AbstractMysqlCDCITBase.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/AbstractMysqlCDCITBase.java @@ -45,6 +45,7 @@ import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; @@ -69,6 +70,15 @@ public abstract class AbstractMysqlCDCITBase extends TestSuiteBase implements Te private final String QUERY_SQL = "select * from %s.%s"; + /** + * Spark reads a CDC source as one long running batch partition without checkpoints, so a sink + * that flushes on checkpoints or full batches does not write the few rows of these jobs. The + * Spark cases in {@link MysqlCDCIT} use jobs that flush every row instead. + */ + protected static final String SPARK_SINK_FLUSH_REASON = + "The jdbc sink of this job only flushes on checkpoints or full batches, and Spark never " + + "checkpoints a CDC source; see the Spark cases in MysqlCDCIT"; + // mysql source table query sql private static final String SOURCE_SQL_TEMPLATE = "select id, cast(f_binary as char) as f_binary, cast(f_blob as char) as f_blob, cast(f_long_varbinary as char) as f_long_varbinary," @@ -143,6 +153,10 @@ public abstract class AbstractMysqlCDCITBase extends TestSuiteBase implements Te } @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = SPARK_SINK_FLUSH_REASON) public void testMysqlCdcCheckDataE2e(TestContainer container) { // Clear related content to ensure that multiple operations are not affected clearTable(MYSQL_DATABASE, SOURCE_TABLE_1); @@ -431,6 +445,10 @@ public abstract class AbstractMysqlCDCITBase extends TestSuiteBase implements Te } @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = SPARK_SINK_FLUSH_REASON) public void testMysqlCdcCheckDataWithDisableExactlyonce(TestContainer container) { // Clear related content to ensure that multiple operations are not affected clearTable(MYSQL_DATABASE, SINK_TABLE); @@ -469,6 +487,10 @@ public abstract class AbstractMysqlCDCITBase extends TestSuiteBase implements Te } @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = SPARK_SINK_FLUSH_REASON) public void testMysqlCdcCheckDataWithNoPrimaryKey(TestContainer container) { // Clear related content to ensure that multiple operations are not affected clearTable(MYSQL_DATABASE, SINK_TABLE); @@ -511,6 +533,10 @@ public abstract class AbstractMysqlCDCITBase extends TestSuiteBase implements Te } @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = SPARK_SINK_FLUSH_REASON) public void testMysqlCdcSnapshotWithNullableUniqueKey(TestContainer container) { // A table without a primary key whose only unique key is nullable: rows with NULL in it // must not be skipped by the snapshot chunks. @@ -1830,6 +1856,10 @@ public abstract class AbstractMysqlCDCITBase extends TestSuiteBase implements Te } @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = SPARK_SINK_FLUSH_REASON) public void testMysqlCdcWithColumnIncludeList(TestContainer container) { // Clear related content to ensure that multiple operations are not affected clearTable(MYSQL_DATABASE, SOURCE_TABLE_1); @@ -1984,6 +2014,89 @@ public abstract class AbstractMysqlCDCITBase extends TestSuiteBase implements Te executeSql("UPDATE " + database + "." + tableName + " SET f_bigint = 10000 where id < 10"); } + /** + * Seeds the source table, then runs a job that copies it to the sink table and checks that the + * sink holds the seeded rows read by the snapshot phase, and afterwards an insert, an update + * and a delete that only the incremental phase can deliver. + */ + protected void checkSnapshotThenIncrementalData(TestContainer container, String confFile) { + clearTable(MYSQL_DATABASE, SOURCE_TABLE_1); + clearTable(MYSQL_DATABASE, SINK_TABLE); + // rows 5, 6 and 8, which the snapshot phase reads + upsertDeleteSourceTable(MYSQL_DATABASE, SOURCE_TABLE_1); + insertCopyOfRow(MYSQL_DATABASE, SOURCE_TABLE_1, 5, 8); + List<List<Object>> snapshotRows = query(getSourceQuerySQL(MYSQL_DATABASE, SOURCE_TABLE_1)); + Assertions.assertEquals(3, snapshotRows.size()); + + CompletableFuture.runAsync( + () -> { + try { + container.executeJob(confFile); + } catch (Exception e) { + log.error("Commit task exception :" + e.getMessage()); + throw new RuntimeException(e); + } + }); + // the budget includes the submission of the job + await().atMost(3, TimeUnit.MINUTES) + .untilAsserted( + () -> + Assertions.assertIterableEquals( + snapshotRows, + query(getSinkQuerySQL(MYSQL_DATABASE, SINK_TABLE)))); + + // insert row 7, update row 5 and delete row 6 after the snapshot phase has been read + insertCopyOfRow(MYSQL_DATABASE, SOURCE_TABLE_1, 5, 7); + executeSql( + "UPDATE " + + MYSQL_DATABASE + + "." + + SOURCE_TABLE_1 + + " SET f_bigint = 20000 where id = 5"); + executeSql("DELETE FROM " + MYSQL_DATABASE + "." + SOURCE_TABLE_1 + " where id = 6"); + List<List<Object>> incrementalRows = + query(getSourceQuerySQL(MYSQL_DATABASE, SOURCE_TABLE_1)); + Assertions.assertNotEquals(snapshotRows, incrementalRows); + + await().atMost(2, TimeUnit.MINUTES) + .untilAsserted( + () -> + Assertions.assertIterableEquals( + incrementalRows, + query(getSinkQuerySQL(MYSQL_DATABASE, SINK_TABLE)))); + } + + /** Inserts a copy of the row {@code sourceId} under the primary key {@code targetId}. */ + private void insertCopyOfRow(String database, String tableName, int sourceId, int targetId) { + String table = database + "." + tableName; + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = + statement.executeQuery("SELECT * FROM " + table + " WHERE 1 = 0")) { + ResultSetMetaData metaData = resultSet.getMetaData(); + List<String> columns = new ArrayList<>(); + List<String> values = new ArrayList<>(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String column = "`" + metaData.getColumnName(i) + "`"; + columns.add(column); + values.add( + "id".equalsIgnoreCase(metaData.getColumnName(i)) + ? String.valueOf(targetId) + : column); + } + statement.execute( + String.format( + "INSERT INTO %s (%s) SELECT %s FROM %s WHERE id = %d", + table, + String.join(", ", columns), + String.join(", ", values), + table, + sourceId)); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + private void upsertDeleteSourceTable(String database, String tableName) { executeSql( diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java index 18011a49919..a46cd7ad686 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java @@ -20,15 +20,14 @@ package org.apache.seatunnel.connectors.seatunnel.cdc.mysql; import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.MySqlVersion; import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.UniqueDatabase; import org.apache.seatunnel.e2e.common.container.EngineType; +import org.apache.seatunnel.e2e.common.container.TestContainer; import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer; +import org.junit.jupiter.api.TestTemplate; + import lombok.extern.slf4j.Slf4j; @Slf4j -@DisabledOnContainer( - value = {}, - type = {EngineType.SPARK}, - disabledReason = "Currently SPARK do not support cdc") public class MysqlCDCIT extends AbstractMysqlCDCITBase { public MysqlCDCIT() { @@ -38,4 +37,25 @@ public class MysqlCDCIT extends AbstractMysqlCDCITBase { new UniqueDatabase( MYSQL_CONTAINER, MYSQL_DATABASE, "mysqluser", "mysqlpw", MYSQL_DATABASE); } + + @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.SEATUNNEL, EngineType.FLINK}, + disabledReason = "Covered by testMysqlCdcCheckDataE2e") + public void testMysqlCdcCheckDataE2eOnSpark(TestContainer container) { + checkSnapshotThenIncrementalData(container, "/mysqlcdc_to_mysql_on_spark.conf"); + } + + @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.SEATUNNEL, EngineType.FLINK}, + disabledReason = + "Covers Spark running all readers of a parallel CDC source in one task, " + + "where the snapshot phase ends without an engine checkpoint") + public void testMysqlCdcCheckDataE2eWithParallelismOnSpark(TestContainer container) { + checkSnapshotThenIncrementalData( + container, "/mysqlcdc_to_mysql_with_parallelism_on_spark.conf"); + } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_on_spark.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_on_spark.conf new file mode 100644 index 00000000000..ca85ec3370a --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_on_spark.conf @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + # You can set engine configuration here + parallelism = 1 + job.mode = "STREAMING" + checkpoint.interval = 5000 + read_limit.bytes_per_second=7000000 + read_limit.rows_per_second=400 +} + +source { + MySQL-CDC { + plugin_output = "customers_mysql_cdc" + server-id = 5652 + username = "st_user_source" + password = "mysqlpw" + table-names = ["mysql_cdc.mysql_cdc_e2e_source_table"] + url = "jdbc:mysql://mysql_cdc_e2e:3306/mysql_cdc" + } +} + +transform { + sql { + plugin_input = "customers_mysql_cdc" + query = """ select id, f_binary, f_blob, f_long_varbinary, f_longblob, f_tinyblob, f_varbinary, f_smallint, f_smallint_unsigned, f_mediumint, + f_mediumint_unsigned, f_int, f_int_unsigned, f_integer, f_integer_unsigned, f_bigint, f_bigint_unsigned, f_numeric, f_decimal, + f_float, f_double, f_double_precision, f_longtext, f_mediumtext, f_text, f_tinytext, f_varchar, f_date, f_datetime, f_timestamp, + f_bit1, f_bit64, f_char, f_enum, f_mediumblob, f_long_varchar, f_real, f_time, f_tinyint, f_tinyint_unsigned, f_json, f_year + from dual """ + plugin_output = "trans_mysql_cdc" + } +} + +sink { + jdbc { + plugin_input = "trans_mysql_cdc" + url = "jdbc:mysql://mysql_cdc_e2e:3306/mysql_cdc" + driver = "com.mysql.cj.jdbc.Driver" + user = "st_user_sink" + password = "mysqlpw" + + generate_sink_sql = true + # You need to configure both database and table + database = mysql_cdc + table = mysql_cdc_e2e_sink_table + primary_keys = ["id"] + # Spark never checkpoints a CDC source, so flush every row instead of waiting for a checkpoint + batch_size = 1 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_parallelism_on_spark.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_parallelism_on_spark.conf new file mode 100644 index 00000000000..03d34f777e4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_parallelism_on_spark.conf @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + # You can set engine configuration here + parallelism = 2 + job.mode = "STREAMING" + checkpoint.interval = 5000 + read_limit.bytes_per_second=7000000 + read_limit.rows_per_second=400 +} + +source { + MySQL-CDC { + plugin_output = "customers_mysql_cdc" + server-id = "5652-5653" + username = "st_user_source" + password = "mysqlpw" + table-names = ["mysql_cdc.mysql_cdc_e2e_source_table"] + url = "jdbc:mysql://mysql_cdc_e2e:3306/mysql_cdc" + # a split size of one row, so the seeded rows give the snapshot phase several splits for the readers + snapshot.split.size = 1 + } +} + +transform { + sql { + plugin_input = "customers_mysql_cdc" + query = """ select id, f_binary, f_blob, f_long_varbinary, f_longblob, f_tinyblob, f_varbinary, f_smallint, f_smallint_unsigned, f_mediumint, + f_mediumint_unsigned, f_int, f_int_unsigned, f_integer, f_integer_unsigned, f_bigint, f_bigint_unsigned, f_numeric, f_decimal, + f_float, f_double, f_double_precision, f_longtext, f_mediumtext, f_text, f_tinytext, f_varchar, f_date, f_datetime, f_timestamp, + f_bit1, f_bit64, f_char, f_enum, f_mediumblob, f_long_varchar, f_real, f_time, f_tinyint, f_tinyint_unsigned, f_json, f_year + from dual """ + plugin_output = "trans_mysql_cdc" + } +} + +sink { + jdbc { + plugin_input = "trans_mysql_cdc" + url = "jdbc:mysql://mysql_cdc_e2e:3306/mysql_cdc" + driver = "com.mysql.cj.jdbc.Driver" + user = "st_user_sink" + password = "mysqlpw" + + generate_sink_sql = true + # You need to configure both database and table + database = mysql_cdc + table = mysql_cdc_e2e_sink_table + primary_keys = ["id"] + # Spark never checkpoints a CDC source, so flush every row instead of waiting for a checkpoint + batch_size = 1 + } +} \ No newline at end of file diff --git a/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/source/CoordinatedEnumeratorContext.java b/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/source/CoordinatedEnumeratorContext.java index 1f79f4691e4..4a58bac00d6 100644 --- a/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/source/CoordinatedEnumeratorContext.java +++ b/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/source/CoordinatedEnumeratorContext.java @@ -42,7 +42,9 @@ public class CoordinatedEnumeratorContext<SplitT extends SourceSplit> @Override public int currentParallelism() { - return coordinatedSource.currentReaderCount(); + // The enumerator is created before the readers, so report the configured parallelism + // rather than the number of readers that exist at the time of the call. + return coordinatedSource.parallelism; } @Override diff --git a/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/source/CoordinatedSource.java b/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/source/CoordinatedSource.java index 472928a3a0d..6ec29bd6d85 100644 --- a/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/source/CoordinatedSource.java +++ b/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/source/CoordinatedSource.java @@ -45,6 +45,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @Slf4j @@ -80,11 +81,20 @@ public class CoordinatedSource<T, SplitT extends SourceSplit, StateT extends Ser protected transient Map<Integer, SourceReader<T, SplitT>> readerMap = new ConcurrentHashMap<>(); protected final Map<Integer, AtomicBoolean> readerRunningMap; protected final AtomicInteger completedReader = new AtomicInteger(0); + protected final AtomicReference<RuntimeException> readerFailure = new AtomicReference<>(); protected transient volatile ScheduledThreadPoolExecutor executorService; /** Flag indicating whether the consumer is still running. */ protected volatile boolean running = true; + /** Flag indicating that {@link #close()} has begun, after which reader failures are ignored. */ + protected volatile boolean closing = false; + + /** + * Id of the last enumerator-only checkpoint, see {@link #getEnumeratorCheckpointInterval()}. + */ + private long enumeratorCheckpointId = 0L; + public CoordinatedSource( SeaTunnelSource<T, SplitT, StateT> source, Map<Integer, List<byte[]>> restoredState, @@ -104,7 +114,28 @@ public class CoordinatedSource<T, SplitT extends SourceSplit, StateT extends Ser createSplitEnumerator(); createReaders(); } catch (Exception e) { - log.warn("create split enumerator or readers failed", e); + RuntimeException failure = + new RuntimeException("Failed to create split enumerator or readers.", e); + closeCreatedResources(failure); + throw failure; + } + } + + /** Releases the enumerator and readers created before the construction failed. */ + private void closeCreatedResources(RuntimeException failure) { + for (SourceReader<T, SplitT> reader : readerMap.values()) { + try { + reader.close(); + } catch (Exception e) { + failure.addSuppressed(e); + } + } + if (splitEnumerator != null) { + try { + splitEnumerator.close(); + } catch (Exception e) { + failure.addSuppressed(e); + } } } @@ -180,15 +211,17 @@ public class CoordinatedSource<T, SplitT extends SourceSplit, StateT extends Ser entry -> { final AtomicBoolean flag = readerRunningMap.get(entry.getKey()); final SourceReader<T, SplitT> reader = entry.getValue(); + final Collector<T> readerCollector = + getReaderCollector(entry.getKey(), collector); executorService.execute( () -> { while (flag.get()) { try { - reader.pollNext(collector); - if (collector.isEmptyThisPollNext()) { + reader.pollNext(readerCollector); + if (readerCollector.isEmptyThisPollNext()) { Thread.sleep(100); } else { - collector.resetEmptyThisPollNext(); + readerCollector.resetEmptyThisPollNext(); /** * sleep(0) is used to prevent the current * thread from occupying CPU resources for a @@ -200,6 +233,15 @@ public class CoordinatedSource<T, SplitT extends SourceSplit, StateT extends Ser Thread.sleep(0L); } } catch (Exception e) { + if (!closing) { + readerFailure.compareAndSet( + null, + new RuntimeException( + String.format( + "SourceReader of subtask %s failed.", + entry.getKey()), + e)); + } running = false; flag.set(false); throw new RuntimeException(e); @@ -208,13 +250,69 @@ public class CoordinatedSource<T, SplitT extends SourceSplit, StateT extends Ser }); }); splitEnumerator.run(); + final long enumeratorCheckpointInterval = getEnumeratorCheckpointInterval(); + long nextEnumeratorCheckpoint = System.currentTimeMillis() + enumeratorCheckpointInterval; while (running) { Thread.sleep(SLEEP_TIME_INTERVAL); + if (!running || closing) { + break; + } + if (enumeratorCheckpointInterval > 0 + && System.currentTimeMillis() >= nextEnumeratorCheckpoint) { + checkpointEnumerator(); + nextEnumeratorCheckpoint = + System.currentTimeMillis() + enumeratorCheckpointInterval; + } } + // The readers run on pooled threads, so their failures must be rethrown here to fail the + // source instead of letting it end as if all readers had completed. + RuntimeException failure = readerFailure.get(); + if (failure != null && !closing) { + throw failure; + } + } + + /** + * Returns the interval in milliseconds of the enumerator-only checkpoints taken by {@link + * #run(Collector)}, or a non-positive value to take none (the default). + * + * <p>Engines that never checkpoint the source can enable them so that enumerators which wait + * for a completed checkpoint before moving on, such as the CDC snapshot split assigner at a + * parallelism above one, still make progress. Only the enumerator takes part: its state is + * snapshotted and discarded, and reader state is neither snapshotted nor committed. + * + * @return the interval in milliseconds, or a non-positive value to disable them + */ + protected long getEnumeratorCheckpointInterval() { + return 0L; + } + + private void checkpointEnumerator() throws Exception { + long checkpointId = ++enumeratorCheckpointId; + try { + splitEnumerator.snapshotState(checkpointId); + splitEnumerator.notifyCheckpointComplete(checkpointId); + } catch (Exception e) { + if (!closing) { + throw e; + } + } + } + + /** + * Returns the collector that the reader of the given subtask emits its records to. + * + * @param subtaskId the subtask index of the reader + * @param collector the collector passed to {@link #run(Collector)} + * @return the collector used by that reader + */ + protected Collector<T> getReaderCollector(int subtaskId, Collector<T> collector) { + return collector; } @Override public void close() throws IOException { + closing = true; running = false; for (Map.Entry<Integer, SourceReader<T, SplitT>> entry : readerMap.entrySet()) { @@ -309,7 +407,6 @@ public class CoordinatedSource<T, SplitT extends SourceSplit, StateT extends Ser protected void handleNoMoreElement(int subtaskId) { readerRunningMap.get(subtaskId).set(false); - readerContextMap.remove(subtaskId); if (completedReader.incrementAndGet() == this.parallelism) { this.running = false; } @@ -327,8 +424,9 @@ public class CoordinatedSource<T, SplitT extends SourceSplit, StateT extends Ser // Enumerator context methods // -------------------------------------------------------------------------------------------- + /** Returns the number of readers that have not signalled the end of their input yet. */ public int currentReaderCount() { - return readerContextMap.size(); + return parallelism - completedReader.get(); } public Set<Integer> registeredReaders() { diff --git a/seatunnel-translation/seatunnel-translation-base/src/test/java/org/apache/seatunnel/translation/source/CoordinatedSourceTest.java b/seatunnel-translation/seatunnel-translation-base/src/test/java/org/apache/seatunnel/translation/source/CoordinatedSourceTest.java new file mode 100644 index 00000000000..ba9f9731683 --- /dev/null +++ b/seatunnel-translation/seatunnel-translation-base/src/test/java/org/apache/seatunnel/translation/source/CoordinatedSourceTest.java @@ -0,0 +1,559 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.translation.source; + +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceEvent; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.source.SourceSplit; +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.source.SupportCoordinate; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +class CoordinatedSourceTest { + + @Test + void shouldExchangeSourceEventsBetweenReadersAndEnumerator() throws Exception { + int parallelism = 2; + List<String> splitIds = + IntStream.range(0, 5).mapToObj(i -> "split-" + i).collect(Collectors.toList()); + EventSource source = new EventSource(splitIds, false); + CoordinatedSource<String, EventSplit, Integer> coordinatedSource = + new CoordinatedSource<>(source, null, parallelism, "coordinated-source-events"); + + ListCollector collector = new ListCollector(); + coordinatedSource.open(); + Assertions.assertTimeoutPreemptively( + Duration.ofSeconds(30), () -> coordinatedSource.run(collector)); + coordinatedSource.close(); + + Assertions.assertEquals( + splitIds, collector.records.stream().sorted().collect(Collectors.toList())); + Assertions.assertEquals( + splitIds, + source.enumerator.finishedSplits.stream().sorted().collect(Collectors.toList())); + Assertions.assertEquals( + splitIds, + source.readers.values().stream() + .flatMap(reader -> reader.ackedSplits.stream()) + .sorted() + .collect(Collectors.toList())); + } + + @Test + void shouldExposeConfiguredParallelismToEnumerator() throws Exception { + int parallelism = 3; + EventSource source = new EventSource(Collections.emptyList(), false); + CoordinatedSource<String, EventSplit, Integer> coordinatedSource = + new CoordinatedSource<>(source, null, parallelism, "coordinated-source-parallel"); + + Assertions.assertEquals(parallelism, source.enumerator.parallelismAtCreation); + Assertions.assertEquals( + parallelism, coordinatedSource.coordinatedEnumeratorContext.currentParallelism()); + coordinatedSource.close(); + } + + @Test + void shouldFailRunWhenReaderFails() throws Exception { + EventSource source = new EventSource(Collections.singletonList("split-0"), true); + CoordinatedSource<String, EventSplit, Integer> coordinatedSource = + new CoordinatedSource<>(source, null, 1, "coordinated-source-failure"); + + coordinatedSource.open(); + Exception exception = + Assertions.assertThrows( + Exception.class, + () -> + Assertions.assertTimeoutPreemptively( + Duration.ofSeconds(30), + () -> coordinatedSource.run(new ListCollector()))); + coordinatedSource.close(); + + Assertions.assertTrue( + hasCause(exception, EventReader.FAILURE_MESSAGE), + "The reader failure must be the cause of the run failure"); + } + + @Test + void shouldFailFastWhenEnumeratorCannotBeCreated() { + EventSource source = new EventSource(Collections.emptyList(), false); + source.failEnumeratorCreation = true; + + RuntimeException exception = + Assertions.assertThrows( + RuntimeException.class, + () -> + new CoordinatedSource<>( + source, null, 1, "coordinated-source-create-failure")); + Assertions.assertTrue(hasCause(exception, EventSource.ENUMERATOR_FAILURE_MESSAGE)); + } + + @Test + void shouldMoveCheckpointGatedEnumeratorOnWithEnumeratorCheckpoints() throws Exception { + int parallelism = 2; + List<String> splitIds = + IntStream.range(0, 4).mapToObj(i -> "split-" + i).collect(Collectors.toList()); + EventSource source = new EventSource(splitIds, false); + source.checkpointGated = true; + CoordinatedSource<String, EventSplit, Integer> coordinatedSource = + new CoordinatedSource<String, EventSplit, Integer>( + source, null, parallelism, "coordinated-source-gated") { + @Override + protected long getEnumeratorCheckpointInterval() { + return 10L; + } + }; + + ListCollector collector = new ListCollector(); + coordinatedSource.open(); + Assertions.assertTimeoutPreemptively( + Duration.ofSeconds(30), () -> coordinatedSource.run(collector)); + coordinatedSource.close(); + + List<String> expected = new ArrayList<>(splitIds); + expected.add(CheckpointGatedEnumerator.FOLLOW_UP_SPLIT); + Assertions.assertEquals( + expected, collector.records.stream().sorted().collect(Collectors.toList())); + } + + @Test + void shouldNotTakeEnumeratorCheckpointsByDefault() throws Exception { + EventSource source = new EventSource(Collections.singletonList("split-0"), false); + CoordinatedSource<String, EventSplit, Integer> coordinatedSource = + new CoordinatedSource<>(source, null, 1, "coordinated-source-default"); + + coordinatedSource.open(); + Assertions.assertTimeoutPreemptively( + Duration.ofSeconds(30), () -> coordinatedSource.run(new ListCollector())); + coordinatedSource.close(); + + Assertions.assertEquals(0, source.enumerator.snapshotCount); + } + + @Test + void shouldIgnoreReaderFailuresCausedByClose() throws Exception { + EventSource source = new EventSource(Collections.emptyList(), false); + source.failReadersOnceClosed = true; + CoordinatedSource<String, EventSplit, Integer> coordinatedSource = + new CoordinatedSource<>(source, null, 1, "coordinated-source-close"); + + coordinatedSource.open(); + Thread runner = + new Thread( + () -> { + try { + coordinatedSource.run(new ListCollector()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + runner.start(); + Assertions.assertTrue(source.readers.get(0).polling.await(30, TimeUnit.SECONDS)); + coordinatedSource.close(); + runner.join(TimeUnit.SECONDS.toMillis(30)); + + Assertions.assertFalse(runner.isAlive()); + Assertions.assertTrue(source.readers.get(0).failedAfterClose.get()); + Assertions.assertNull(coordinatedSource.readerFailure.get()); + } + + @Test + void shouldCloseCreatedReadersAndEnumeratorWhenConstructionFails() { + EventSource source = new EventSource(Collections.emptyList(), false); + source.failReaderCreationForSubtask = 1; + + Assertions.assertThrows( + RuntimeException.class, + () -> new CoordinatedSource<>(source, null, 2, "coordinated-source-cleanup")); + Assertions.assertTrue(source.enumerator.closed); + Assertions.assertTrue(source.readers.get(0).closed); + } + + private static boolean hasCause(Throwable throwable, String message) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (message.equals(current.getMessage())) { + return true; + } + } + return false; + } + + private static final class ListCollector implements Collector<String> { + + private final Object lock = new Object(); + private final Queue<String> records = new ConcurrentLinkedQueue<>(); + + @Override + public void collect(String record) { + records.add(record); + } + + @Override + public Object getCheckpointLock() { + return lock; + } + } + + /** A source whose readers report every finished split to the enumerator, like CDC sources. */ + private static final class EventSource + implements SeaTunnelSource<String, EventSplit, Integer>, SupportCoordinate { + + private static final String ENUMERATOR_FAILURE_MESSAGE = "cannot create enumerator"; + + private final List<String> splitIds; + private final boolean failReaders; + private final Map<Integer, EventReader> readers = new ConcurrentHashMap<>(); + private boolean failEnumeratorCreation; + private boolean checkpointGated; + private boolean failReadersOnceClosed; + private int failReaderCreationForSubtask = -1; + private EventEnumerator enumerator; + + private EventSource(List<String> splitIds, boolean failReaders) { + this.splitIds = splitIds; + this.failReaders = failReaders; + } + + @Override + public Boundedness getBoundedness() { + return Boundedness.BOUNDED; + } + + @Override + public SourceReader<String, EventSplit> createReader(SourceReader.Context readerContext) { + if (readerContext.getIndexOfSubtask() == failReaderCreationForSubtask) { + throw new IllegalStateException("cannot create reader"); + } + EventReader reader = new EventReader(readerContext, failReaders); + reader.failOnceClosed = failReadersOnceClosed; + readers.put(readerContext.getIndexOfSubtask(), reader); + return reader; + } + + @Override + public SourceSplitEnumerator<EventSplit, Integer> createEnumerator( + SourceSplitEnumerator.Context<EventSplit> enumeratorContext) { + if (failEnumeratorCreation) { + throw new IllegalStateException(ENUMERATOR_FAILURE_MESSAGE); + } + enumerator = + checkpointGated + ? new CheckpointGatedEnumerator(enumeratorContext, splitIds) + : new EventEnumerator(enumeratorContext, splitIds); + return enumerator; + } + + @Override + public SourceSplitEnumerator<EventSplit, Integer> restoreEnumerator( + SourceSplitEnumerator.Context<EventSplit> enumeratorContext, + Integer checkpointState) { + return createEnumerator(enumeratorContext); + } + + @Override + public String getPluginName() { + return "coordinated-event-source"; + } + } + + private static class EventEnumerator implements SourceSplitEnumerator<EventSplit, Integer> { + + protected final SourceSplitEnumerator.Context<EventSplit> context; + private final int parallelismAtCreation; + protected final Queue<String> pendingSplits; + protected final Queue<String> finishedSplits = new ConcurrentLinkedQueue<>(); + protected volatile int snapshotCount; + private volatile boolean closed; + + private EventEnumerator( + SourceSplitEnumerator.Context<EventSplit> context, List<String> splitIds) { + this.context = context; + this.parallelismAtCreation = context.currentParallelism(); + this.pendingSplits = new ConcurrentLinkedQueue<>(splitIds); + } + + @Override + public void open() {} + + @Override + public void run() {} + + @Override + public void close() { + closed = true; + } + + @Override + public void addSplitsBack(List<EventSplit> splits, int subtaskId) { + splits.forEach(split -> pendingSplits.add(split.splitId())); + } + + @Override + public int currentUnassignedSplitSize() { + return pendingSplits.size(); + } + + @Override + public synchronized void handleSplitRequest(int subtaskId) { + String splitId = pendingSplits.poll(); + if (splitId == null) { + context.signalNoMoreSplits(subtaskId); + } else { + context.assignSplit(subtaskId, new EventSplit(splitId)); + } + } + + @Override + public void registerReader(int subtaskId) {} + + @Override + public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { + String splitId = ((SplitFinishedEvent) sourceEvent).splitId; + finishedSplits.add(splitId); + context.sendEventToSourceReader(subtaskId, new SplitAckEvent(splitId)); + } + + @Override + public Integer snapshotState(long checkpointId) { + snapshotCount++; + return null; + } + + @Override + public void notifyCheckpointComplete(long checkpointId) {} + } + + /** + * Hands out a follow-up split only after every split has finished and a checkpoint taken after + * that has completed, like the CDC snapshot split assigner at a parallelism above one. + */ + private static final class CheckpointGatedEnumerator extends EventEnumerator { + + private static final String FOLLOW_UP_SPLIT = "split-follow-up"; + + private final int splitCount; + private final Set<Integer> awaitingReaders = new TreeSet<>(); + private long checkpointToComplete = -1L; + private boolean gateOpen; + private boolean followUpAssigned; + + private CheckpointGatedEnumerator( + SourceSplitEnumerator.Context<EventSplit> context, List<String> splitIds) { + super(context, splitIds); + this.splitCount = splitIds.size(); + } + + @Override + public synchronized void handleSplitRequest(int subtaskId) { + awaitingReaders.add(subtaskId); + assignSplits(); + } + + @Override + public synchronized Integer snapshotState(long checkpointId) { + super.snapshotState(checkpointId); + if (checkpointToComplete < 0 && finishedSplits.size() == splitCount) { + checkpointToComplete = checkpointId; + } + return null; + } + + @Override + public synchronized void notifyCheckpointComplete(long checkpointId) { + if (checkpointToComplete >= 0 && checkpointId >= checkpointToComplete) { + gateOpen = true; + assignSplits(); + } + } + + private void assignSplits() { + Iterator<Integer> awaiting = awaitingReaders.iterator(); + while (awaiting.hasNext()) { + int subtaskId = awaiting.next(); + String splitId = pendingSplits.poll(); + if (splitId == null && !gateOpen) { + return; + } + if (splitId == null && !followUpAssigned) { + followUpAssigned = true; + splitId = FOLLOW_UP_SPLIT; + } + if (splitId == null) { + context.signalNoMoreSplits(subtaskId); + } else { + context.assignSplit(subtaskId, new EventSplit(splitId)); + } + awaiting.remove(); + } + } + } + + private static final class EventReader implements SourceReader<String, EventSplit> { + + private static final String FAILURE_MESSAGE = "reader failed"; + + private final SourceReader.Context context; + private final boolean fail; + private final Queue<EventSplit> splits = new ConcurrentLinkedQueue<>(); + private final List<String> ackedSplits = Collections.synchronizedList(new ArrayList<>()); + private final CountDownLatch polling = new CountDownLatch(1); + private final AtomicBoolean failedAfterClose = new AtomicBoolean(); + private volatile boolean splitRequested; + private volatile boolean noMoreSplits; + private volatile boolean failOnceClosed; + private volatile boolean closed; + + private EventReader(SourceReader.Context context, boolean fail) { + this.context = context; + this.fail = fail; + } + + @Override + public void open() {} + + @Override + public void close() throws IOException { + closed = true; + if (failOnceClosed) { + // wait until the polling thread has failed because of this close + long deadline = System.currentTimeMillis() + 30_000L; + while (!failedAfterClose.get() && System.currentTimeMillis() < deadline) { + sleep(5L); + } + // let the polling thread finish handling its failure + sleep(200L); + } + } + + @Override + public void pollNext(Collector<String> output) throws Exception { + polling.countDown(); + if (fail) { + throw new IllegalStateException(FAILURE_MESSAGE); + } + if (failOnceClosed) { + // like a reader blocked in I/O that fails once its connection is closed + while (!closed) { + sleep(5L); + } + failedAfterClose.set(true); + throw new IllegalStateException("reader is closed"); + } + EventSplit split = splits.poll(); + if (split != null) { + output.collect(split.splitId()); + // the enumerator answers with an ack, which requests the next split + context.sendSourceEventToEnumerator(new SplitFinishedEvent(split.splitId())); + return; + } + if (noMoreSplits) { + context.signalNoMoreElement(); + } else if (!splitRequested) { + splitRequested = true; + context.sendSplitRequest(); + } + } + + @Override + public List<EventSplit> snapshotState(long checkpointId) { + return new ArrayList<>(splits); + } + + @Override + public void addSplits(List<EventSplit> splits) { + this.splits.addAll(splits); + } + + @Override + public void handleNoMoreSplits() { + noMoreSplits = true; + } + + @Override + public void handleSourceEvent(SourceEvent sourceEvent) { + ackedSplits.add(((SplitAckEvent) sourceEvent).splitId); + splitRequested = false; + } + + @Override + public void notifyCheckpointComplete(long checkpointId) {} + } + + private static void sleep(long millis) throws IOException { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + } + + private static final class EventSplit implements SourceSplit { + + private final String splitId; + + private EventSplit(String splitId) { + this.splitId = splitId; + } + + @Override + public String splitId() { + return splitId; + } + } + + private static final class SplitFinishedEvent implements SourceEvent { + + private final String splitId; + + private SplitFinishedEvent(String splitId) { + this.splitId = splitId; + } + } + + private static final class SplitAckEvent implements SourceEvent { + + private final String splitId; + + private SplitAckEvent(String splitId) { + this.splitId = splitId; + } + } +} diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceSupport.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceSupport.java index 6c072836c8d..0c3fc6337a5 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceSupport.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/SeaTunnelSourceSupport.java @@ -21,6 +21,7 @@ import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils; import org.apache.seatunnel.api.options.EnvCommonOptions; import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SupportCoordinate; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; @@ -93,6 +94,16 @@ public class SeaTunnelSourceSupport String checkpointLocation, DataSourceOptions options) { SeaTunnelSource<SeaTunnelRow, ?, ?> seaTunnelSource = getSeaTunnelSource(options); + if (seaTunnelSource instanceof SupportCoordinate) { + throw new UnsupportedOperationException( + String.format( + "Source [%s] coordinates its readers through source events (SupportCoordinate), " + + "which Spark micro-batch reading does not support yet because it " + + "cannot restore the source state between micro-batches. Read it with " + + "Spark batch reading, which the SeaTunnel Spark starter uses for both " + + "BATCH and STREAMING jobs, or run the job on Zeta or Flink.", + seaTunnelSource.getPluginName())); + } Integer parallelism = options.getInt(EnvCommonOptions.PARALLELISM.key(), 1); String applicationId = SparkSession.getActiveSession().get().sparkContext().applicationId(); Integer checkpointInterval = diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/CoordinatedBatchPartitionReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/CoordinatedBatchPartitionReader.java index 47d55767ff4..03a283b4d4e 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/CoordinatedBatchPartitionReader.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-2.4/src/main/java/org/apache/seatunnel/translation/spark/source/reader/batch/CoordinatedBatchPartitionReader.java @@ -19,7 +19,6 @@ package org.apache.seatunnel.translation.spark.source.reader.batch; import org.apache.seatunnel.api.source.Collector; import org.apache.seatunnel.api.source.SeaTunnelSource; -import org.apache.seatunnel.api.source.SourceReader; import org.apache.seatunnel.api.source.SourceSplit; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.translation.source.BaseSourceFunction; @@ -31,10 +30,11 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; public class CoordinatedBatchPartitionReader extends ParallelBatchPartitionReader { + private static final long ENUMERATOR_CHECKPOINT_INTERVAL_MS = 1000L; + protected final Map<Integer, InternalRowCollector> collectorMap; public CoordinatedBatchPartitionReader( @@ -75,47 +75,21 @@ public class CoordinatedBatchPartitionReader extends ParallelBatchPartitionReade } @Override - public void run(Collector<SeaTunnelRow> collector) throws Exception { - readerMap - .entrySet() - .parallelStream() - .forEach( - entry -> { - final AtomicBoolean flag = readerRunningMap.get(entry.getKey()); - final SourceReader<SeaTunnelRow, SplitT> reader = entry.getValue(); - final Collector<SeaTunnelRow> rowCollector = - collectorMap.get(entry.getKey()); - executorService.execute( - () -> { - while (flag.get()) { - try { - reader.pollNext(rowCollector); - if (rowCollector.isEmptyThisPollNext()) { - Thread.sleep(100); - } else { - rowCollector.resetEmptyThisPollNext(); - /** - * sleep(0) is used to prevent the current - * thread from occupying CPU resources for a - * long time, thus blocking the checkpoint - * thread for a long time. It is mentioned - * in this - * https://github.com/apache/seatunnel/issues/5694 - */ - Thread.sleep(0L); - } - } catch (Exception e) { - this.running = false; - flag.set(false); - throw new RuntimeException(e); - } - } - }); - }); - splitEnumerator.run(); - while (this.running) { - Thread.sleep(SLEEP_TIME_INTERVAL); - } + protected Collector<SeaTunnelRow> getReaderCollector( + int subtaskId, Collector<SeaTunnelRow> collector) { + return collectorMap.get(subtaskId); + } + + /** + * The batch read never checkpoints the source, so checkpoint the enumerator alone to let + * enumerators that wait for a completed checkpoint, such as the CDC snapshot split assigner + * at a parallelism above one, move on. Every reader of this source emits into the same + * handover queue of this single partition, so records still reach the sink in the order + * they were emitted. + */ + @Override + protected long getEnumeratorCheckpointInterval() { + return ENUMERATOR_CHECKPOINT_INTERVAL_MS; } @Override diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReader.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReader.java index 69dc64c234b..5e2941ded3b 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReader.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReader.java @@ -19,7 +19,6 @@ package org.apache.seatunnel.translation.spark.source.partition.batch; import org.apache.seatunnel.api.source.Collector; import org.apache.seatunnel.api.source.SeaTunnelSource; -import org.apache.seatunnel.api.source.SourceReader; import org.apache.seatunnel.api.source.SourceSplit; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.translation.source.BaseSourceFunction; @@ -31,10 +30,11 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; public class CoordinatedBatchPartitionReader extends ParallelBatchPartitionReader { + private static final long ENUMERATOR_CHECKPOINT_INTERVAL_MS = 1000L; + protected final Map<Integer, InternalRowCollector> collectorMap; public CoordinatedBatchPartitionReader( @@ -75,47 +75,21 @@ public class CoordinatedBatchPartitionReader extends ParallelBatchPartitionReade } @Override - public void run(Collector<SeaTunnelRow> collector) throws Exception { - readerMap - .entrySet() - .parallelStream() - .forEach( - entry -> { - final AtomicBoolean flag = readerRunningMap.get(entry.getKey()); - final SourceReader<SeaTunnelRow, SplitT> reader = entry.getValue(); - final Collector<SeaTunnelRow> rowCollector = - collectorMap.get(entry.getKey()); - executorService.execute( - () -> { - while (flag.get()) { - try { - reader.pollNext(rowCollector); - if (rowCollector.isEmptyThisPollNext()) { - Thread.sleep(100); - } else { - rowCollector.resetEmptyThisPollNext(); - /** - * sleep(0) is used to prevent the current - * thread from occupying CPU resources for a - * long time, thus blocking the checkpoint - * thread for a long time. It is mentioned - * in this - * https://github.com/apache/seatunnel/issues/5694 - */ - Thread.sleep(0L); - } - } catch (Exception e) { - this.running = false; - flag.set(false); - throw new RuntimeException(e); - } - } - }); - }); - splitEnumerator.run(); - while (this.running) { - Thread.sleep(SLEEP_TIME_INTERVAL); - } + protected Collector<SeaTunnelRow> getReaderCollector( + int subtaskId, Collector<SeaTunnelRow> collector) { + return collectorMap.get(subtaskId); + } + + /** + * The batch read never checkpoints the source, so checkpoint the enumerator alone to let + * enumerators that wait for a completed checkpoint, such as the CDC snapshot split assigner + * at a parallelism above one, move on. Every reader of this source emits into the same + * handover queue of this single partition, so records still reach the sink in the order + * they were emitted. + */ + @Override + protected long getEnumeratorCheckpointInterval() { + return ENUMERATOR_CHECKPOINT_INTERVAL_MS; } @Override diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScan.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScan.java index c2085d91270..a2069271a23 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScan.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/main/java/org/apache/seatunnel/translation/spark/source/scan/SeaTunnelScan.java @@ -18,6 +18,7 @@ package org.apache.seatunnel.translation.spark.source.scan; import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SupportCoordinate; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.translation.spark.execution.MultiTableManager; import org.apache.seatunnel.translation.spark.source.partition.batch.SeaTunnelBatch; @@ -68,6 +69,16 @@ public class SeaTunnelScan implements Scan { @Override public MicroBatchStream toMicroBatchStream(String checkpointLocation) { + if (source instanceof SupportCoordinate) { + throw new UnsupportedOperationException( + String.format( + "Source [%s] coordinates its readers through source events (SupportCoordinate), " + + "which Spark micro-batch reading does not support yet because it " + + "cannot restore the source state between micro-batches. Read it with " + + "Spark batch reading, which the SeaTunnel Spark starter uses for both " + + "BATCH and STREAMING jobs, or run the job on Zeta or Flink.", + source.getPluginName())); + } return new SeaTunnelMicroBatch( source, parallelism, diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReaderTest.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReaderTest.java new file mode 100644 index 00000000000..b6aa23133df --- /dev/null +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/source/partition/batch/CoordinatedBatchPartitionReaderTest.java @@ -0,0 +1,433 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.translation.spark.source.partition.batch; + +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceEvent; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.source.SourceSplit; +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.source.SupportCoordinate; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.translation.spark.execution.MultiTableManager; +import org.apache.seatunnel.translation.spark.source.scan.SeaTunnelScan; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.read.InputPartition; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +class CoordinatedBatchPartitionReaderTest { + + @Test + void shouldReadSourceThatExchangesEventsBetweenReadersAndEnumerator() throws Exception { + int parallelism = 2; + List<String> splitIds = + IntStream.range(0, 4).mapToObj(i -> "split-" + i).collect(Collectors.toList()); + EventSource source = new EventSource(splitIds, false); + + InputPartition[] partitions = + new SeaTunnelBatch( + source, + parallelism, + "coordinated-batch-events", + Collections.emptyMap(), + multiTableManager()) + .planInputPartitions(); + Assertions.assertEquals(1, partitions.length); + + List<String> rows = new ArrayList<>(); + MultiTableManager multiTableManager = multiTableManager(); + CoordinatedBatchPartitionReader partitionReader = + new CoordinatedBatchPartitionReader( + source, + parallelism, + "coordinated-batch-events", + 0, + Collections.emptyMap(), + multiTableManager); + Assertions.assertTimeoutPreemptively( + Duration.ofSeconds(30), + () -> { + while (partitionReader.next()) { + InternalRow row = partitionReader.get(); + if (row != null) { + rows.add((String) multiTableManager.reconvert(row).getField(0)); + } + } + }); + partitionReader.close(); + + Assertions.assertEquals(splitIds, rows.stream().sorted().collect(Collectors.toList())); + Assertions.assertEquals( + splitIds, source.finishedSplits.stream().sorted().collect(Collectors.toList())); + } + + @Test + void shouldFailPartitionWhenReaderFails() throws Exception { + EventSource source = new EventSource(Collections.singletonList("split-0"), true); + CoordinatedBatchPartitionReader partitionReader = + new CoordinatedBatchPartitionReader( + source, + 1, + "coordinated-batch-failure", + 0, + Collections.emptyMap(), + multiTableManager()); + + Exception exception = + Assertions.assertThrows( + Exception.class, + () -> + Assertions.assertTimeoutPreemptively( + Duration.ofSeconds(30), + () -> { + while (partitionReader.next()) { + partitionReader.get(); + } + })); + partitionReader.close(); + + boolean causedByReader = false; + for (Throwable cause = exception; cause != null; cause = cause.getCause()) { + causedByReader |= EventReader.FAILURE_MESSAGE.equals(cause.getMessage()); + } + Assertions.assertTrue(causedByReader, "The reader failure must fail the partition"); + } + + @Test + void shouldMoveCheckpointGatedEnumeratorOnWithoutEngineCheckpoints() throws Exception { + int parallelism = 2; + List<String> splitIds = + IntStream.range(0, 4).mapToObj(i -> "split-" + i).collect(Collectors.toList()); + EventSource source = new EventSource(splitIds, false); + source.checkpointGated = true; + + MultiTableManager multiTableManager = multiTableManager(); + CoordinatedBatchPartitionReader partitionReader = + new CoordinatedBatchPartitionReader( + source, + parallelism, + "coordinated-batch-gated", + 0, + Collections.emptyMap(), + multiTableManager); + List<String> rows = new ArrayList<>(); + Assertions.assertTimeoutPreemptively( + Duration.ofSeconds(60), + () -> { + while (partitionReader.next()) { + InternalRow row = partitionReader.get(); + if (row != null) { + rows.add((String) multiTableManager.reconvert(row).getField(0)); + } + } + }); + partitionReader.close(); + + List<String> expected = new ArrayList<>(splitIds); + expected.add(EventEnumerator.FOLLOW_UP_SPLIT); + Assertions.assertEquals(expected, rows.stream().sorted().collect(Collectors.toList())); + } + + @Test + void shouldRejectCoordinatedSourceForMicroBatchReading() { + SeaTunnelScan scan = + new SeaTunnelScan( + new EventSource(Collections.emptyList(), false), + 1, + "coordinated-micro-batch", + new CaseInsensitiveStringMap(Collections.emptyMap()), + multiTableManager()); + + UnsupportedOperationException exception = + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> scan.toMicroBatchStream("/tmp/unused-checkpoint")); + Assertions.assertTrue(exception.getMessage().contains("coordinated-event-source")); + } + + private static MultiTableManager multiTableManager() { + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"value"}, new SeaTunnelDataType[] {BasicType.STRING_TYPE}); + return new MultiTableManager( + new CatalogTable[] {CatalogTableUtil.getCatalogTable("events", rowType)}); + } + + /** A source whose readers report every finished split to the enumerator, like CDC sources. */ + private static final class EventSource + implements SeaTunnelSource<SeaTunnelRow, EventSplit, Integer>, SupportCoordinate { + + private final List<String> splitIds; + private final boolean failReaders; + private final Queue<String> finishedSplits = new ConcurrentLinkedQueue<>(); + private boolean checkpointGated; + + private EventSource(List<String> splitIds, boolean failReaders) { + this.splitIds = splitIds; + this.failReaders = failReaders; + } + + @Override + public Boundedness getBoundedness() { + return Boundedness.BOUNDED; + } + + @Override + public SourceReader<SeaTunnelRow, EventSplit> createReader( + SourceReader.Context readerContext) { + return new EventReader(readerContext, failReaders); + } + + @Override + public SourceSplitEnumerator<EventSplit, Integer> createEnumerator( + SourceSplitEnumerator.Context<EventSplit> enumeratorContext) { + return new EventEnumerator( + enumeratorContext, splitIds, finishedSplits, checkpointGated); + } + + @Override + public SourceSplitEnumerator<EventSplit, Integer> restoreEnumerator( + SourceSplitEnumerator.Context<EventSplit> enumeratorContext, + Integer checkpointState) { + return createEnumerator(enumeratorContext); + } + + @Override + public String getPluginName() { + return "coordinated-event-source"; + } + } + + /** + * Assigns one split per request. When checkpoint gated it hands out a follow-up split only + * after every split has finished and a checkpoint taken after that has completed, like the CDC + * snapshot split assigner at a parallelism above one. + */ + private static final class EventEnumerator + implements SourceSplitEnumerator<EventSplit, Integer> { + + private static final String FOLLOW_UP_SPLIT = "split-follow-up"; + + private final SourceSplitEnumerator.Context<EventSplit> context; + private final Queue<String> pendingSplits; + private final Queue<String> finishedSplits; + private final int splitCount; + private final boolean checkpointGated; + private final Set<Integer> awaitingReaders = new TreeSet<>(); + private long checkpointToComplete = -1L; + private boolean gateOpen; + private boolean followUpAssigned; + + private EventEnumerator( + SourceSplitEnumerator.Context<EventSplit> context, + List<String> splitIds, + Queue<String> finishedSplits, + boolean checkpointGated) { + this.context = context; + this.pendingSplits = new ConcurrentLinkedQueue<>(splitIds); + this.finishedSplits = finishedSplits; + this.splitCount = splitIds.size(); + this.checkpointGated = checkpointGated; + } + + @Override + public void open() {} + + @Override + public void run() {} + + @Override + public void close() {} + + @Override + public void addSplitsBack(List<EventSplit> splits, int subtaskId) { + splits.forEach(split -> pendingSplits.add(split.splitId())); + } + + @Override + public int currentUnassignedSplitSize() { + return pendingSplits.size(); + } + + @Override + public synchronized void handleSplitRequest(int subtaskId) { + awaitingReaders.add(subtaskId); + assignSplits(); + } + + private void assignSplits() { + Iterator<Integer> awaiting = awaitingReaders.iterator(); + while (awaiting.hasNext()) { + int subtaskId = awaiting.next(); + String splitId = pendingSplits.poll(); + if (splitId == null && checkpointGated && !gateOpen) { + return; + } + if (splitId == null && checkpointGated && !followUpAssigned) { + followUpAssigned = true; + splitId = FOLLOW_UP_SPLIT; + } + if (splitId == null) { + context.signalNoMoreSplits(subtaskId); + } else { + context.assignSplit(subtaskId, new EventSplit(splitId)); + } + awaiting.remove(); + } + } + + @Override + public void registerReader(int subtaskId) {} + + @Override + public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { + String splitId = ((SplitFinishedEvent) sourceEvent).splitId; + finishedSplits.add(splitId); + context.sendEventToSourceReader(subtaskId, new SplitAckEvent()); + } + + @Override + public synchronized Integer snapshotState(long checkpointId) { + if (checkpointToComplete < 0 && finishedSplits.size() == splitCount) { + checkpointToComplete = checkpointId; + } + return null; + } + + @Override + public synchronized void notifyCheckpointComplete(long checkpointId) { + if (checkpointToComplete >= 0 && checkpointId >= checkpointToComplete) { + gateOpen = true; + assignSplits(); + } + } + } + + private static final class EventReader implements SourceReader<SeaTunnelRow, EventSplit> { + + private static final String FAILURE_MESSAGE = "reader failed"; + + private final SourceReader.Context context; + private final boolean fail; + private final Queue<EventSplit> splits = new ConcurrentLinkedQueue<>(); + private volatile boolean splitRequested; + private volatile boolean noMoreSplits; + + private EventReader(SourceReader.Context context, boolean fail) { + this.context = context; + this.fail = fail; + } + + @Override + public void open() {} + + @Override + public void close() {} + + @Override + public void pollNext(Collector<SeaTunnelRow> output) { + if (fail) { + throw new IllegalStateException(FAILURE_MESSAGE); + } + EventSplit split = splits.poll(); + if (split != null) { + output.collect(new SeaTunnelRow(new Object[] {split.splitId()})); + // the enumerator answers with an ack, which requests the next split + context.sendSourceEventToEnumerator(new SplitFinishedEvent(split.splitId())); + return; + } + if (noMoreSplits) { + context.signalNoMoreElement(); + } else if (!splitRequested) { + splitRequested = true; + context.sendSplitRequest(); + } + } + + @Override + public List<EventSplit> snapshotState(long checkpointId) { + return new ArrayList<>(splits); + } + + @Override + public void addSplits(List<EventSplit> splits) { + this.splits.addAll(splits); + } + + @Override + public void handleNoMoreSplits() { + noMoreSplits = true; + } + + @Override + public void handleSourceEvent(SourceEvent sourceEvent) { + splitRequested = false; + } + + @Override + public void notifyCheckpointComplete(long checkpointId) {} + } + + private static final class EventSplit implements SourceSplit { + + private final String splitId; + + private EventSplit(String splitId) { + this.splitId = splitId; + } + + @Override + public String splitId() { + return splitId; + } + } + + private static final class SplitFinishedEvent implements SourceEvent { + + private final String splitId; + + private SplitFinishedEvent(String splitId) { + this.splitId = splitId; + } + } + + private static final class SplitAckEvent implements SourceEvent {} +}
