This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-13201-1f61649c24fe8a081ddf5bde136664f3cd7fe7f8 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit b8d67bbd94567ba8c97e49ba56e587a539661c3b Author: zhiweiniu <[email protected]> AuthorDate: Thu Nov 12 07:51:26 2026 +0000 [Improve][API] Run multi-table timer flushes in parallel (#13201) --- docs/en/engines/zeta/timer-flush.md | 10 +- docs/zh/engines/zeta/timer-flush.md | 9 +- .../api/sink/multitablesink/MultiTableSink.java | 5 +- .../sink/multitablesink/MultiTableSinkWriter.java | 233 ++++++++++++++---- .../multitablesink/MultiTableSinkWriterTest.java | 269 +++++++++++++++++---- 5 files changed, 425 insertions(+), 101 deletions(-) diff --git a/docs/en/engines/zeta/timer-flush.md b/docs/en/engines/zeta/timer-flush.md index 0ba3bef4a7..d710096c9e 100644 --- a/docs/en/engines/zeta/timer-flush.md +++ b/docs/en/engines/zeta/timer-flush.md @@ -179,11 +179,15 @@ responsible for any buffered data that has not been flushed by a timer. For a Multi-Table Sink, the Sink registers one aggregated action when at least one child writer supports timer flush. Before invoking child actions, the Multi-Table writer waits for its internal row queues and in-progress writes to finish. This prevents timer flush from racing with its -asynchronous writer threads. +asynchronous writer threads. After the drain, child writers flush in parallel across queues and +sequentially within each queue. On success, the Sink waits for every queue before processing later +records. The configured Multi-Table failure policy also applies to timer flush. `FAIL_FAST` propagates the -first child failure. A policy that continues other tables isolates the failed table and continues -flushing healthy tables. +first reported child failure promptly. A failure that ends the round or a cancellation stops +the queues from starting further child flush actions. During shutdown, each queue's writers close +after its in-flight flush exits. Shared resources are released after all writers close. A policy +that continues other tables isolates the failed table and continues flushing healthy tables. ## Monitoring diff --git a/docs/zh/engines/zeta/timer-flush.md b/docs/zh/engines/zeta/timer-flush.md index ef3686c179..b738dbc3db 100644 --- a/docs/zh/engines/zeta/timer-flush.md +++ b/docs/zh/engines/zeta/timer-flush.md @@ -161,10 +161,13 @@ Connector 提供的幂等能力或确定性键。 对于 Multi-Table Sink,只要至少一个子 Writer 支持定时刷新,Sink 就会注册一个聚合操作。 调用子 Writer 操作前,Multi-Table Writer 会等待内部数据队列和正在执行的写入完成,避免定时 -刷新与异步写线程发生竞争。 +刷新与异步写线程发生竞争。排空后,不同队列并行刷新,同一队列内的子 Writer 依次刷新。 +刷新成功时,Sink 等待各队列结束后再处理后续记录。 -定时刷新也遵循已配置的 Multi-Table 失败策略。`FAIL_FAST` 会传递第一个子 Writer 异常; -允许继续处理其他表的策略会隔离失败表,并继续刷新健康表。 +定时刷新也遵循已配置的 Multi-Table 失败策略。`FAIL_FAST` 收到首个子 Writer 异常后立即传递。 +刷新因失败或取消而结束后,各队列停止启动新的子 Writer 刷新。Sink 关闭时,每个队列会等本队列 +正在执行的刷新退出后再关闭 Writer。所有 Writer 关闭后才释放共享资源。允许继续处理其他表的策略 +会隔离失败表,并继续刷新健康表。 ## 监控指标 diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java index 34bf0dc07f..36e8cd52c4 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java @@ -259,8 +259,9 @@ public class MultiTableSink * Registers an aggregated flush action on the parent context if any sub-writer registered a * flush action via its {@link SinkContextProxy}. * - * <p>The registered action drains all blocking queues and then calls each sub-writer's flush - * action under the corresponding lock, ensuring safe execution from the engine timer thread. + * <p>The sink task invokes the action when it consumes a flush signal. The action drains + * pending writes and flushes independent queues in parallel, with child writers serialized + * under their queue lock. */ private void registerAggregatedFlushIfNeeded( SinkWriter.Context context, diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java index 6037894acd..271f6df30e 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java @@ -31,6 +31,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.tracing.MDCTracer; import org.apache.seatunnel.common.constants.JobMode; import org.apache.seatunnel.common.utils.HashUtils; +import org.apache.seatunnel.common.utils.function.RunnableWithException; import lombok.extern.slf4j.Slf4j; @@ -46,14 +47,18 @@ import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -66,10 +71,9 @@ import java.util.stream.Collectors; * appropriate sub-writer. Routing is based on the row's primary key hash for ordered delivery, or * random selection when no primary key exists. * - * <p>Thread pool sizing is {@code queueSize * 2}: half for queue consumer threads, half for - * parallel {@link #prepareCommit(long)} tasks. Synchronization between the consumer threads and - * checkpoint/commit operations is coordinated through locks on each {@link - * MultiTableWriterRunnable} instance. + * <p>Thread pool sizing is {@code queueSize * 2}, with half for queue consumer threads and half for + * parallel timer flush and {@link #prepareCommit(long)} tasks. These operations synchronize with + * the consumer threads through locks on each {@link MultiTableWriterRunnable} instance. */ @Slf4j public class MultiTableSinkWriter @@ -77,6 +81,8 @@ public class MultiTableSinkWriter SupportSchemaEvolutionSinkWriter { private static final long EXECUTOR_CLOSE_TIMEOUT_SECONDS = 60L; + private static final CompletableFuture<Throwable> TIMER_FLUSH_CLOSED = + CompletableFuture.completedFuture(null); private final Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters; private final Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext; @@ -89,6 +95,8 @@ public class MultiTableSinkWriter private final List<BlockingQueue<MultiTableWriterRunnable.QueueElement>> blockingQueues = new ArrayList<>(); private final ExecutorService executorService; + private final AtomicReference<CompletableFuture<Throwable>> activeTimerFlush = + new AtomicReference<>(); private final MultiTableFailurePolicy failurePolicy; private final JobMode jobMode; private final int tableRetryTimes; @@ -108,9 +116,8 @@ public class MultiTableSinkWriter * * <p>Initializes a fixed thread pool of size {@code queueSize * 2}: half the threads run {@link * MultiTableWriterRunnable} consumers that drain the blocking queues, and the other half are - * reserved for parallel {@link #prepareCommit(long)} tasks. Each sub-writer is assigned to a - * queue using the formula {@code sinkIdentifier.getIndex() % queueSize}, ensuring deterministic - * distribution across queues. + * reserved for timer flush and parallel {@link #prepareCommit(long)} tasks. Each sub-writer is + * assigned to a queue using the formula {@code sinkIdentifier.getIndex() % queueSize}. * * @param sinkWriters map of sink identifiers to their corresponding writers * @param queueSize number of blocking queues (and consumer threads) to create @@ -204,9 +211,7 @@ public class MultiTableSinkWriter executorService = MDCTracer.tracing( Executors.newFixedThreadPool( - // we use it in `MultiTableWriterRunnable` and `prepare commit - // task`, so it - // should be double. + // Reserve one thread per queue for flush and prepare commit tasks. queueSize * 2, runnable -> { Thread thread = new Thread(runnable); @@ -746,12 +751,15 @@ public class MultiTableSinkWriter subSinkErrorCheck(); MultiTableCommitInfo multiTableCommitInfo = new MultiTableCommitInfo(new ConcurrentHashMap<>()); + ConcurrentMap<String, Throwable> checkpointFailures = new ConcurrentHashMap<>(); List<Future<?>> futures = new ArrayList<>(); for (int i = 0; i < sinkWritersWithIndex.size(); i++) { int subWriterIndex = i; futures.add( executorService.submit( () -> { + String fatalFailureTable = null; + Throwable fatalFailure = null; synchronized (runnable.get(subWriterIndex)) { for (Map.Entry<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriterEntry : @@ -759,15 +767,19 @@ public class MultiTableSinkWriter sinkWritersWithIndex .get(subWriterIndex) .entrySet())) { + SinkIdentifier sinkIdentifier = sinkWriterEntry.getKey(); + String tableId = sinkIdentifier.getTableIdentifier(); + if (isTableFailed(tableId) + || checkpointFailures.containsKey(tableId)) { + continue; + } Optional<?> commit; try { SinkWriter<SeaTunnelRow, ?, ?> sinkWriter = sinkWriterEntry.getValue(); commit = executeWithTableRetry( - sinkWriterEntry - .getKey() - .getTableIdentifier(), + tableId, MultiTableFailurePhase.CHECKPOINT, () -> sinkWriter.prepareCommit( @@ -776,24 +788,30 @@ public class MultiTableSinkWriter Thread.currentThread().interrupt(); throw new RuntimeException(error); } catch (Throwable error) { - if (failurePolicy.continueOtherTables()) { - handleTableFailure( - sinkWriterEntry - .getKey() - .getTableIdentifier(), - MultiTableFailurePhase.CHECKPOINT, - error); - continue; + if (!failurePolicy.continueOtherTables()) { + throw new RuntimeException(error); } - throw new RuntimeException(error); + if (containsFatalRowErrorHandlingFailure(error)) { + fatalFailureTable = tableId; + fatalFailure = error; + break; + } + checkpointFailures.putIfAbsent(tableId, error); + continue; } commit.ifPresent( o -> multiTableCommitInfo .getCommitInfo() - .put(sinkWriterEntry.getKey(), o)); + .put(sinkIdentifier, o)); } } + if (fatalFailure != null) { + handleTableFailure( + fatalFailureTable, + MultiTableFailurePhase.CHECKPOINT, + fatalFailure); + } })); } for (Future<?> future : futures) { @@ -803,6 +821,10 @@ public class MultiTableSinkWriter throw new RuntimeException(e); } } + // Table isolation acquires other queue locks. Run it after the prepare tasks finish. + checkpointFailures.forEach( + (tableId, error) -> + handleTableFailure(tableId, MultiTableFailurePhase.CHECKPOINT, error)); waitRuntimeTableFailuresHandled(); subSinkErrorCheck(); if (multiTableCommitInfo.getCommitInfo().isEmpty()) { @@ -850,21 +872,24 @@ public class MultiTableSinkWriter /** * Closes this writer and releases all resources. * - * <p>Drains remaining queues, then calls {@link ExecutorService#shutdownNow()} to interrupt all - * consumer threads. Each sub-writer is closed under its respective {@link - * MultiTableWriterRunnable} lock to avoid concurrent access. Finally, the shared {@link + * <p>Signals any active timer flush round to stop, drains pending writes, and interrupts the + * executor threads. After any in-progress table isolation finishes, each sub-writer is closed + * under its queue lock, which also protects any in-flight flush. Finally, the shared {@link * MultiTableResourceManager} is closed and this method waits for the executor threads to stop. * * <p>Uses first-exception-wins error handling: if multiple sub-writers throw during close, only * the first exception is propagated. Resource manager close errors are logged but not * propagated. * - * @throws RuntimeException wrapping the first {@link Throwable} caught from any sub-writer; - * note that the method signature declares {@code throws IOException} (inherited from the - * interface), but the current implementation always wraps in {@code RuntimeException} + * @throws IOException if shutdown reports failed tables for a batch job + * @throws RuntimeException if shutdown fails without a batch failed-table report */ @Override public void close() throws IOException { + CompletableFuture<Throwable> flush = activeTimerFlush.getAndSet(TIMER_FLUSH_CLOSED); + if (flush != null && flush != TIMER_FLUSH_CLOSED) { + flush.complete(new IOException("Multi-table sink writer is closing")); + } // The variables used in lambda expressions should be final or valid final, so they are // modified to arrays final Throwable[] firstE = {null}; @@ -875,6 +900,21 @@ public class MultiTableSinkWriter firstE[0] = e; } executorService.shutdownNow(); + if (failurePolicy.continueOtherTables()) { + synchronized (this) { + // When a timer flush round is cancelled, a worker may already be isolating + // a failed table. That isolation can continue after shutdownNow() returns + // while the worker closes and removes the table's writers across queues. + // + // Queue locks serialize individual close calls. If shutdown closes a writer + // first, the writer is still in the map and isolation could close it again. + // + // handleTableFailure() holds this monitor. A flush worker checks the round's + // stop signal under the same monitor before starting isolation. Since close() + // has already signaled the round to stop, acquiring this monitor waits for any + // isolation in progress. Release it before acquiring the queue locks below. + } + } for (int i = 0; i < sinkWritersWithIndex.size(); i++) { synchronized (runnable.get(i)) { Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkIdentifierSinkWriterMap = @@ -947,37 +987,130 @@ public class MultiTableSinkWriter } /** - * Aggregated flush triggered by the engine timer. Drains all blocking queues first, then calls - * each sub-writer's flush action under the corresponding {@link MultiTableWriterRunnable} lock - * to prevent concurrent writes. + * Drains the row queues and flushes independent queues in parallel. A successful round waits + * for every queue. A fatal failure ends the round and prevents further child flush actions. * * @param proxyContexts map from sink identifier to its {@link SinkContextProxy} */ void aggregatedFlush(Map<SinkIdentifier, SinkContextProxy> proxyContexts) throws Exception { checkQueueRemain(); subSinkErrorCheck(); - for (int i = 0; i < sinkWritersWithIndex.size(); i++) { - synchronized (runnable.get(i)) { - for (SinkIdentifier id : sinkWritersWithIndex.get(i).keySet()) { + // A null result means all queues succeeded. Any completed result stops new actions. + CompletableFuture<Throwable> completion = new CompletableFuture<>(); + if (!activeTimerFlush.compareAndSet(null, completion)) { + throw new IOException( + "Multi-table timer flush is already running or the writer is closing"); + } + List<Future<?>> flushTasks = new ArrayList<>(); + boolean succeeded = false; + try { + if (sinkWritersWithIndex.size() == 1) { + flushQueue(0, proxyContexts, completion); + completion.complete(null); + } else { + try { + submitQueueFlushes(proxyContexts, completion, flushTasks); + } catch (RejectedExecutionException error) { + // Closing can shut down the executor after the submit loop checks completion. + completion.complete(error); + } + } + Throwable failure = completion.get(); + if (failure instanceof Exception) { + throw (Exception) failure; + } + if (failure instanceof Error) { + throw (Error) failure; + } + if (failure != null) { + throw new ExecutionException(failure); + } + succeeded = true; + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw error; + } finally { + if (!succeeded) { + completion.cancel(false); + flushTasks.forEach(task -> task.cancel(true)); + } + activeTimerFlush.compareAndSet(completion, null); + } + } + + private void submitQueueFlushes( + Map<SinkIdentifier, SinkContextProxy> proxyContexts, + CompletableFuture<Throwable> completion, + List<Future<?>> flushTasks) { + int queueCount = sinkWritersWithIndex.size(); + AtomicInteger remainingQueues = new AtomicInteger(queueCount); + for (int i = 0; i < queueCount && !completion.isDone(); i++) { + int queueIndex = i; + flushTasks.add( + executorService.submit( + () -> { + try { + flushQueue(queueIndex, proxyContexts, completion); + if (remainingQueues.decrementAndGet() == 0) { + completion.complete(null); + } + } catch (Throwable error) { + if (!completion.complete(error)) { + log.warn( + "Timer flush task for queue {} failed after the round ended", + queueIndex, + error); + } + } + })); + } + } + + private void flushQueue( + int queueIndex, + Map<SinkIdentifier, SinkContextProxy> proxyContexts, + CompletableFuture<Throwable> completion) { + MultiTableWriterRunnable writerRunnable = runnable.get(queueIndex); + for (SinkIdentifier id : sinkWritersWithIndex.get(queueIndex).keySet()) { + try { + synchronized (writerRunnable) { + if (completion.isDone()) { + return; + } + if (isTableFailed(id.getTableIdentifier())) { + continue; + } SinkContextProxy proxy = proxyContexts.get(id); - if (proxy != null && proxy.getFlushAction() != null) { - try { - proxy.getFlushAction().run(); - } catch (InterruptedException error) { - Thread.currentThread().interrupt(); - throw error; - } catch (Exception error) { - if (failurePolicy.continueOtherTables()) { - handleTableFailure( - id.getTableIdentifier(), - MultiTableFailurePhase.TIMER_FLUSH, - error); - continue; - } - throw error; + RunnableWithException flushAction = + proxy == null ? null : proxy.getFlushAction(); + if (flushAction != null) { + flushAction.run(); + } + } + } catch (Throwable error) { + if (error instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } else if (error instanceof Exception && failurePolicy.continueOtherTables()) { + // Table isolation acquires other queue locks. Coordinate with close outside + // ours. + synchronized (this) { + if (!completion.isDone()) { + handleTableFailure( + id.getTableIdentifier(), + MultiTableFailurePhase.TIMER_FLUSH, + error); + continue; } } } + if (!completion.complete(error)) { + log.warn( + "Timer flush failed for table {} in queue {} after the round ended", + id.getTableIdentifier(), + queueIndex, + error); + } + return; } } } diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java index 6035c52b4d..4e6c3c4acb 100644 --- a/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java +++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java @@ -47,6 +47,7 @@ import lombok.Data; import java.io.IOException; import java.io.Serializable; import java.lang.reflect.Field; +import java.time.Duration; import java.util.Base64; import java.util.Collections; import java.util.HashMap; @@ -54,11 +55,14 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -488,6 +492,61 @@ public class MultiTableSinkWriterTest { Assertions.assertDoesNotThrow(multiTableSinkWriter::close); } + @Test + public void testPrepareCommitIsolatesConcurrentFailures() { + Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new HashMap<>(); + Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new HashMap<>(); + SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy", 2); + sinkWriters.put(healthyIdentifier, new TestSinkWriter()); + sinkWritersContext.put(healthyIdentifier, new TestSinkWriterContext()); + + CountDownLatch preparesStarted = new CountDownLatch(2); + AtomicInteger failedPrepares = new AtomicInteger(); + for (int queueIndex = 0; queueIndex < 2; queueIndex++) { + SinkIdentifier failedIdentifier = + SinkIdentifier.of("test.failed" + queueIndex, queueIndex); + sinkWriters.put( + failedIdentifier, + new TestSinkWriter() { + @Override + public Optional<TestSinkState> prepareCommit(long checkpointId) + throws IOException { + preparesStarted.countDown(); + try { + Assertions.assertTrue(preparesStarted.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + failedPrepares.incrementAndGet(); + throw new IOException("intentional prepare failure"); + } + }); + sinkWritersContext.put(failedIdentifier, new TestSinkWriterContext()); + } + + MultiTableSinkWriter multiTableSinkWriter = + new MultiTableSinkWriter( + sinkWriters, + 2, + sinkWritersContext, + MultiTableFailurePolicy.CONTINUE_OTHER_TABLES, + JobMode.BATCH); + MultiTableCommitInfo commitInfo = + Assertions.assertTimeoutPreemptively( + Duration.ofSeconds(10), + () -> multiTableSinkWriter.prepareCommit(1L).orElseThrow()); + Assertions.assertEquals(2, failedPrepares.get()); + Assertions.assertEquals( + Collections.singleton(healthyIdentifier), commitInfo.getCommitInfo().keySet()); + + IOException closeException = + Assertions.assertThrows(IOException.class, multiTableSinkWriter::close); + Assertions.assertTrue(closeException.getMessage().contains("test.failed0")); + Assertions.assertTrue(closeException.getMessage().contains("test.failed1")); + Assertions.assertTrue(closeException.getMessage().contains("phase=checkpoint")); + } + @Test public void testPrepareCommitFatalRowErrorHandlingFailureFailsFast() { Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new HashMap<>(); @@ -570,28 +629,35 @@ public class MultiTableSinkWriterTest { } @Test - public void testAggregatedFlushIsolatesFailedTableAndFlushesHealthyTable() { + public void testAggregatedFlushIsolatesConcurrentFailuresAndFlushesHealthyTable() { Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new HashMap<>(); Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new HashMap<>(); Map<SinkIdentifier, SinkContextProxy> proxyContexts = new HashMap<>(); - SinkIdentifier failedIdentifier = SinkIdentifier.of("test.failed", 0); - SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy", 1); - sinkWriters.put(failedIdentifier, new TestSinkWriter()); + SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy", 2); sinkWriters.put(healthyIdentifier, new TestSinkWriter()); - sinkWritersContext.put(failedIdentifier, new TestSinkWriterContext()); sinkWritersContext.put(healthyIdentifier, new TestSinkWriterContext()); AtomicInteger failedFlushCount = new AtomicInteger(); - SinkContextProxy failedProxy = new SinkContextProxy(0, 2, new TestSinkWriterContext()); - failedProxy.registerFlushAction( - () -> { - failedFlushCount.incrementAndGet(); - throw new IOException("intentional flush failure"); - }); + CountDownLatch failedFlushesStarted = new CountDownLatch(2); + for (int queueIndex = 0; queueIndex < 2; queueIndex++) { + SinkIdentifier failedIdentifier = + SinkIdentifier.of("test.failed" + queueIndex, queueIndex); + sinkWriters.put(failedIdentifier, new TestSinkWriter()); + sinkWritersContext.put(failedIdentifier, new TestSinkWriterContext()); + SinkContextProxy failedProxy = + new SinkContextProxy(queueIndex, 2, new TestSinkWriterContext()); + failedProxy.registerFlushAction( + () -> { + failedFlushCount.incrementAndGet(); + failedFlushesStarted.countDown(); + Assertions.assertTrue(failedFlushesStarted.await(5, TimeUnit.SECONDS)); + throw new IOException("intentional flush failure"); + }); + proxyContexts.put(failedIdentifier, failedProxy); + } AtomicInteger healthyFlushCount = new AtomicInteger(); - SinkContextProxy healthyProxy = new SinkContextProxy(1, 2, new TestSinkWriterContext()); + SinkContextProxy healthyProxy = new SinkContextProxy(2, 2, new TestSinkWriterContext()); healthyProxy.registerFlushAction(healthyFlushCount::incrementAndGet); - proxyContexts.put(failedIdentifier, failedProxy); proxyContexts.put(healthyIdentifier, healthyProxy); MultiTableSinkWriter multiTableSinkWriter = @@ -602,61 +668,178 @@ public class MultiTableSinkWriterTest { MultiTableFailurePolicy.CONTINUE_OTHER_TABLES, JobMode.BATCH); - Assertions.assertDoesNotThrow(() -> multiTableSinkWriter.aggregatedFlush(proxyContexts)); - Assertions.assertEquals(1, failedFlushCount.get()); + Assertions.assertTimeoutPreemptively( + Duration.ofSeconds(10), () -> multiTableSinkWriter.aggregatedFlush(proxyContexts)); + Assertions.assertEquals(2, failedFlushCount.get()); Assertions.assertEquals(1, healthyFlushCount.get()); Assertions.assertDoesNotThrow(() -> multiTableSinkWriter.aggregatedFlush(proxyContexts)); - Assertions.assertEquals(1, failedFlushCount.get()); + Assertions.assertEquals(2, failedFlushCount.get()); Assertions.assertEquals(2, healthyFlushCount.get()); IOException closeException = Assertions.assertThrows(IOException.class, multiTableSinkWriter::close); - Assertions.assertTrue(closeException.getMessage().contains("test.failed")); + Assertions.assertTrue(closeException.getMessage().contains("test.failed0")); + Assertions.assertTrue(closeException.getMessage().contains("test.failed1")); Assertions.assertTrue(closeException.getMessage().contains("phase=timer_flush")); } @Test - public void testAggregatedFlushFailFastStopsAtFirstFailure() { + public void testAggregatedFlushReportsFailureAndClosesQueuesIndependently() throws Exception { Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new HashMap<>(); Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new HashMap<>(); Map<SinkIdentifier, SinkContextProxy> proxyContexts = new HashMap<>(); SinkIdentifier failedIdentifier = SinkIdentifier.of("test.failed", 0); - SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy", 1); - sinkWriters.put(failedIdentifier, new TestSinkWriter()); - sinkWriters.put(healthyIdentifier, new TestSinkWriter()); + SinkIdentifier slowIdentifier = SinkIdentifier.of("test.slow", 1); + SinkIdentifier siblingIdentifier = SinkIdentifier.of("test.sibling", 3); + CountDownLatch failedWriterClosed = new CountDownLatch(1); + AtomicBoolean slowFlushFinished = new AtomicBoolean(); + AtomicBoolean slowWriterClosed = new AtomicBoolean(); + sinkWriters.put( + failedIdentifier, + new TestSinkWriter() { + @Override + public void close() { + failedWriterClosed.countDown(); + } + }); + sinkWriters.put( + slowIdentifier, + new TestSinkWriter() { + @Override + public void close() { + Assertions.assertTrue(slowFlushFinished.get()); + slowWriterClosed.set(true); + } + }); + sinkWriters.put(siblingIdentifier, new TestSinkWriter()); + sinkWritersContext.put(slowIdentifier, new TestSinkWriterContext()); + sinkWritersContext.put(siblingIdentifier, new TestSinkWriterContext()); sinkWritersContext.put(failedIdentifier, new TestSinkWriterContext()); - sinkWritersContext.put(healthyIdentifier, new TestSinkWriterContext()); - AtomicInteger failedFlushCount = new AtomicInteger(); + AtomicInteger slowQueueFlushCount = new AtomicInteger(); + CountDownLatch slowFlushStarted = new CountDownLatch(1); + CompletableFuture<Void> releaseSlowFlush = new CompletableFuture<>(); + IOException expectedFailure = new IOException("intentional flush failure"); + SinkContextProxy slowProxy = new SinkContextProxy(1, 2, new TestSinkWriterContext()); + slowProxy.registerFlushAction( + () -> { + // Block whichever child is visited first on this queue. + if (slowQueueFlushCount.incrementAndGet() == 1) { + slowFlushStarted.countDown(); + releaseSlowFlush.join(); + slowFlushFinished.set(true); + } + }); + SinkContextProxy siblingProxy = new SinkContextProxy(3, 2, new TestSinkWriterContext()); + siblingProxy.registerFlushAction(slowProxy.getFlushAction()); SinkContextProxy failedProxy = new SinkContextProxy(0, 2, new TestSinkWriterContext()); failedProxy.registerFlushAction( () -> { - failedFlushCount.incrementAndGet(); - throw new IOException("intentional flush failure"); + Assertions.assertTrue(slowFlushStarted.await(5, TimeUnit.SECONDS)); + throw expectedFailure; }); - AtomicInteger healthyFlushCount = new AtomicInteger(); - SinkContextProxy healthyProxy = new SinkContextProxy(1, 2, new TestSinkWriterContext()); - healthyProxy.registerFlushAction(healthyFlushCount::incrementAndGet); + proxyContexts.put(slowIdentifier, slowProxy); + proxyContexts.put(siblingIdentifier, siblingProxy); proxyContexts.put(failedIdentifier, failedProxy); - proxyContexts.put(healthyIdentifier, healthyProxy); MultiTableSinkWriter multiTableSinkWriter = - new MultiTableSinkWriter( - sinkWriters, - 2, - sinkWritersContext, - MultiTableFailurePolicy.FAIL_FAST, - JobMode.BATCH); + new MultiTableSinkWriter(sinkWriters, 2, sinkWritersContext); + FutureTask<Void> flush = + new FutureTask<>( + () -> { + multiTableSinkWriter.aggregatedFlush(proxyContexts); + return null; + }); + Thread dispatcher = new Thread(flush); + dispatcher.setDaemon(true); + FutureTask<Void> close = + new FutureTask<>( + () -> { + multiTableSinkWriter.close(); + return null; + }); + Thread closer = new Thread(close); + closer.setDaemon(true); + try { + multiTableSinkWriter.write(buildRow("test.failed", 0)); + dispatcher.start(); + ExecutionException error = + Assertions.assertThrows( + ExecutionException.class, () -> flush.get(5, TimeUnit.SECONDS)); + Assertions.assertSame(expectedFailure, error.getCause()); - IOException flushException = - Assertions.assertThrows( - IOException.class, - () -> multiTableSinkWriter.aggregatedFlush(proxyContexts)); - Assertions.assertEquals("intentional flush failure", flushException.getMessage()); - Assertions.assertEquals(1, failedFlushCount.get()); - Assertions.assertEquals(0, healthyFlushCount.get()); - Assertions.assertDoesNotThrow(multiTableSinkWriter::close); + closer.start(); + Assertions.assertTrue(failedWriterClosed.await(5, TimeUnit.SECONDS)); + Assertions.assertFalse(slowWriterClosed.get()); + } finally { + releaseSlowFlush.complete(null); + dispatcher.join(5000); + if (closer.getState() == Thread.State.NEW) { + closer.start(); + } + close.get(5, TimeUnit.SECONDS); + } + Assertions.assertTrue(slowWriterClosed.get()); + Assertions.assertEquals(1, slowQueueFlushCount.get()); + } + + @Test + public void testInterruptedAggregatedFlushDoesNotStartAnotherChildFlush() throws Exception { + Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new HashMap<>(); + Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new HashMap<>(); + Map<SinkIdentifier, SinkContextProxy> proxyContexts = new HashMap<>(); + AtomicInteger startedFlushes = new AtomicInteger(); + CountDownLatch firstFlushStarted = new CountDownLatch(1); + CompletableFuture<Void> releaseFirstFlush = new CompletableFuture<>(); + for (int index : new int[] {0, 2}) { + SinkIdentifier identifier = SinkIdentifier.of("test.table" + index, index); + SinkWriter.Context context = new TestSinkWriterContext(); + sinkWriters.put(identifier, new TestSinkWriter()); + sinkWritersContext.put(identifier, context); + SinkContextProxy proxy = new SinkContextProxy(index, 2, context); + proxy.registerFlushAction( + () -> { + if (startedFlushes.incrementAndGet() == 1) { + firstFlushStarted.countDown(); + releaseFirstFlush.join(); + // Simulate a connector that consumes the interrupt during I/O. + Thread.interrupted(); + } + }); + proxyContexts.put(identifier, proxy); + } + + MultiTableSinkWriter multiTableSinkWriter = + new MultiTableSinkWriter(sinkWriters, 2, sinkWritersContext); + AtomicBoolean dispatcherInterrupted = new AtomicBoolean(); + FutureTask<Void> flush = + new FutureTask<>( + () -> { + try { + multiTableSinkWriter.aggregatedFlush(proxyContexts); + return null; + } finally { + dispatcherInterrupted.set(Thread.currentThread().isInterrupted()); + } + }); + Thread dispatcher = new Thread(flush); + dispatcher.setDaemon(true); + dispatcher.start(); + try { + Assertions.assertTrue(firstFlushStarted.await(5, TimeUnit.SECONDS)); + dispatcher.interrupt(); + ExecutionException error = + Assertions.assertThrows( + ExecutionException.class, () -> flush.get(5, TimeUnit.SECONDS)); + Assertions.assertTrue(error.getCause() instanceof InterruptedException); + Assertions.assertTrue(dispatcherInterrupted.get()); + } finally { + releaseFirstFlush.complete(null); + dispatcher.join(5000); + multiTableSinkWriter.close(); + } + Assertions.assertEquals(1, startedFlushes.get()); } @Test
