This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/dev/pr-13201-1f61649c24fe8a081ddf5bde136664f3cd7fe7f8
in repository https://gitbox.apache.org/repos/asf/seatunnel.git

commit b8d67bbd94567ba8c97e49ba56e587a539661c3b
Author: zhiweiniu <[email protected]>
AuthorDate: Thu Nov 12 07:51:26 2026 +0000

    [Improve][API] Run multi-table timer flushes in parallel (#13201)
---
 docs/en/engines/zeta/timer-flush.md                |  10 +-
 docs/zh/engines/zeta/timer-flush.md                |   9 +-
 .../api/sink/multitablesink/MultiTableSink.java    |   5 +-
 .../sink/multitablesink/MultiTableSinkWriter.java  | 233 ++++++++++++++----
 .../multitablesink/MultiTableSinkWriterTest.java   | 269 +++++++++++++++++----
 5 files changed, 425 insertions(+), 101 deletions(-)

diff --git a/docs/en/engines/zeta/timer-flush.md 
b/docs/en/engines/zeta/timer-flush.md
index 0ba3bef4a7..d710096c9e 100644
--- a/docs/en/engines/zeta/timer-flush.md
+++ b/docs/en/engines/zeta/timer-flush.md
@@ -179,11 +179,15 @@ responsible for any buffered data that has not been 
flushed by a timer.
 For a Multi-Table Sink, the Sink registers one aggregated action when at least 
one child writer
 supports timer flush. Before invoking child actions, the Multi-Table writer 
waits for its internal
 row queues and in-progress writes to finish. This prevents timer flush from 
racing with its
-asynchronous writer threads.
+asynchronous writer threads. After the drain, child writers flush in parallel 
across queues and
+sequentially within each queue. On success, the Sink waits for every queue 
before processing later
+records.
 
 The configured Multi-Table failure policy also applies to timer flush. 
`FAIL_FAST` propagates the
-first child failure. A policy that continues other tables isolates the failed 
table and continues
-flushing healthy tables.
+first reported child failure promptly. A failure that ends the round or a 
cancellation stops
+the queues from starting further child flush actions. During shutdown, each 
queue's writers close
+after its in-flight flush exits. Shared resources are released after all 
writers close. A policy
+that continues other tables isolates the failed table and continues flushing 
healthy tables.
 
 ## Monitoring
 
diff --git a/docs/zh/engines/zeta/timer-flush.md 
b/docs/zh/engines/zeta/timer-flush.md
index ef3686c179..b738dbc3db 100644
--- a/docs/zh/engines/zeta/timer-flush.md
+++ b/docs/zh/engines/zeta/timer-flush.md
@@ -161,10 +161,13 @@ Connector 提供的幂等能力或确定性键。
 
 对于 Multi-Table Sink,只要至少一个子 Writer 支持定时刷新,Sink 就会注册一个聚合操作。
 调用子 Writer 操作前,Multi-Table Writer 会等待内部数据队列和正在执行的写入完成,避免定时
-刷新与异步写线程发生竞争。
+刷新与异步写线程发生竞争。排空后,不同队列并行刷新,同一队列内的子 Writer 依次刷新。
+刷新成功时,Sink 等待各队列结束后再处理后续记录。
 
-定时刷新也遵循已配置的 Multi-Table 失败策略。`FAIL_FAST` 会传递第一个子 Writer 异常;
-允许继续处理其他表的策略会隔离失败表,并继续刷新健康表。
+定时刷新也遵循已配置的 Multi-Table 失败策略。`FAIL_FAST` 收到首个子 Writer 异常后立即传递。
+刷新因失败或取消而结束后,各队列停止启动新的子 Writer 刷新。Sink 关闭时,每个队列会等本队列
+正在执行的刷新退出后再关闭 Writer。所有 Writer 关闭后才释放共享资源。允许继续处理其他表的策略
+会隔离失败表,并继续刷新健康表。
 
 ## 监控指标
 
diff --git 
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java
 
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java
index 34bf0dc07f..36e8cd52c4 100644
--- 
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java
+++ 
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java
@@ -259,8 +259,9 @@ public class MultiTableSink
      * Registers an aggregated flush action on the parent context if any 
sub-writer registered a
      * flush action via its {@link SinkContextProxy}.
      *
-     * <p>The registered action drains all blocking queues and then calls each 
sub-writer's flush
-     * action under the corresponding lock, ensuring safe execution from the 
engine timer thread.
+     * <p>The sink task invokes the action when it consumes a flush signal. 
The action drains
+     * pending writes and flushes independent queues in parallel, with child 
writers serialized
+     * under their queue lock.
      */
     private void registerAggregatedFlushIfNeeded(
             SinkWriter.Context context,
diff --git 
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
 
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
index 6037894acd..271f6df30e 100644
--- 
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
+++ 
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
@@ -31,6 +31,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow;
 import org.apache.seatunnel.api.tracing.MDCTracer;
 import org.apache.seatunnel.common.constants.JobMode;
 import org.apache.seatunnel.common.utils.HashUtils;
+import org.apache.seatunnel.common.utils.function.RunnableWithException;
 
 import lombok.extern.slf4j.Slf4j;
 
@@ -46,14 +47,18 @@ import java.util.Optional;
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
@@ -66,10 +71,9 @@ import java.util.stream.Collectors;
  * appropriate sub-writer. Routing is based on the row's primary key hash for 
ordered delivery, or
  * random selection when no primary key exists.
  *
- * <p>Thread pool sizing is {@code queueSize * 2}: half for queue consumer 
threads, half for
- * parallel {@link #prepareCommit(long)} tasks. Synchronization between the 
consumer threads and
- * checkpoint/commit operations is coordinated through locks on each {@link
- * MultiTableWriterRunnable} instance.
+ * <p>Thread pool sizing is {@code queueSize * 2}, with half for queue 
consumer threads and half for
+ * parallel timer flush and {@link #prepareCommit(long)} tasks. These 
operations synchronize with
+ * the consumer threads through locks on each {@link MultiTableWriterRunnable} 
instance.
  */
 @Slf4j
 public class MultiTableSinkWriter
@@ -77,6 +81,8 @@ public class MultiTableSinkWriter
                 SupportSchemaEvolutionSinkWriter {
 
     private static final long EXECUTOR_CLOSE_TIMEOUT_SECONDS = 60L;
+    private static final CompletableFuture<Throwable> TIMER_FLUSH_CLOSED =
+            CompletableFuture.completedFuture(null);
 
     private final Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> 
sinkWriters;
     private final Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext;
@@ -89,6 +95,8 @@ public class MultiTableSinkWriter
     private final List<BlockingQueue<MultiTableWriterRunnable.QueueElement>> 
blockingQueues =
             new ArrayList<>();
     private final ExecutorService executorService;
+    private final AtomicReference<CompletableFuture<Throwable>> 
activeTimerFlush =
+            new AtomicReference<>();
     private final MultiTableFailurePolicy failurePolicy;
     private final JobMode jobMode;
     private final int tableRetryTimes;
@@ -108,9 +116,8 @@ public class MultiTableSinkWriter
      *
      * <p>Initializes a fixed thread pool of size {@code queueSize * 2}: half 
the threads run {@link
      * MultiTableWriterRunnable} consumers that drain the blocking queues, and 
the other half are
-     * reserved for parallel {@link #prepareCommit(long)} tasks. Each 
sub-writer is assigned to a
-     * queue using the formula {@code sinkIdentifier.getIndex() % queueSize}, 
ensuring deterministic
-     * distribution across queues.
+     * reserved for timer flush and parallel {@link #prepareCommit(long)} 
tasks. Each sub-writer is
+     * assigned to a queue using the formula {@code sinkIdentifier.getIndex() 
% queueSize}.
      *
      * @param sinkWriters map of sink identifiers to their corresponding 
writers
      * @param queueSize number of blocking queues (and consumer threads) to 
create
@@ -204,9 +211,7 @@ public class MultiTableSinkWriter
         executorService =
                 MDCTracer.tracing(
                         Executors.newFixedThreadPool(
-                                // we use it in `MultiTableWriterRunnable` and 
`prepare commit
-                                // task`, so it
-                                // should be double.
+                                // Reserve one thread per queue for flush and 
prepare commit tasks.
                                 queueSize * 2,
                                 runnable -> {
                                     Thread thread = new Thread(runnable);
@@ -746,12 +751,15 @@ public class MultiTableSinkWriter
         subSinkErrorCheck();
         MultiTableCommitInfo multiTableCommitInfo =
                 new MultiTableCommitInfo(new ConcurrentHashMap<>());
+        ConcurrentMap<String, Throwable> checkpointFailures = new 
ConcurrentHashMap<>();
         List<Future<?>> futures = new ArrayList<>();
         for (int i = 0; i < sinkWritersWithIndex.size(); i++) {
             int subWriterIndex = i;
             futures.add(
                     executorService.submit(
                             () -> {
+                                String fatalFailureTable = null;
+                                Throwable fatalFailure = null;
                                 synchronized (runnable.get(subWriterIndex)) {
                                     for (Map.Entry<SinkIdentifier, 
SinkWriter<SeaTunnelRow, ?, ?>>
                                             sinkWriterEntry :
@@ -759,15 +767,19 @@ public class MultiTableSinkWriter
                                                             
sinkWritersWithIndex
                                                                     
.get(subWriterIndex)
                                                                     
.entrySet())) {
+                                        SinkIdentifier sinkIdentifier = 
sinkWriterEntry.getKey();
+                                        String tableId = 
sinkIdentifier.getTableIdentifier();
+                                        if (isTableFailed(tableId)
+                                                || 
checkpointFailures.containsKey(tableId)) {
+                                            continue;
+                                        }
                                         Optional<?> commit;
                                         try {
                                             SinkWriter<SeaTunnelRow, ?, ?> 
sinkWriter =
                                                     sinkWriterEntry.getValue();
                                             commit =
                                                     executeWithTableRetry(
-                                                            sinkWriterEntry
-                                                                    .getKey()
-                                                                    
.getTableIdentifier(),
+                                                            tableId,
                                                             
MultiTableFailurePhase.CHECKPOINT,
                                                             () ->
                                                                     
sinkWriter.prepareCommit(
@@ -776,24 +788,30 @@ public class MultiTableSinkWriter
                                             Thread.currentThread().interrupt();
                                             throw new RuntimeException(error);
                                         } catch (Throwable error) {
-                                            if 
(failurePolicy.continueOtherTables()) {
-                                                handleTableFailure(
-                                                        sinkWriterEntry
-                                                                .getKey()
-                                                                
.getTableIdentifier(),
-                                                        
MultiTableFailurePhase.CHECKPOINT,
-                                                        error);
-                                                continue;
+                                            if 
(!failurePolicy.continueOtherTables()) {
+                                                throw new 
RuntimeException(error);
                                             }
-                                            throw new RuntimeException(error);
+                                            if 
(containsFatalRowErrorHandlingFailure(error)) {
+                                                fatalFailureTable = tableId;
+                                                fatalFailure = error;
+                                                break;
+                                            }
+                                            
checkpointFailures.putIfAbsent(tableId, error);
+                                            continue;
                                         }
                                         commit.ifPresent(
                                                 o ->
                                                         multiTableCommitInfo
                                                                 
.getCommitInfo()
-                                                                
.put(sinkWriterEntry.getKey(), o));
+                                                                
.put(sinkIdentifier, o));
                                     }
                                 }
+                                if (fatalFailure != null) {
+                                    handleTableFailure(
+                                            fatalFailureTable,
+                                            MultiTableFailurePhase.CHECKPOINT,
+                                            fatalFailure);
+                                }
                             }));
         }
         for (Future<?> future : futures) {
@@ -803,6 +821,10 @@ public class MultiTableSinkWriter
                 throw new RuntimeException(e);
             }
         }
+        // Table isolation acquires other queue locks. Run it after the 
prepare tasks finish.
+        checkpointFailures.forEach(
+                (tableId, error) ->
+                        handleTableFailure(tableId, 
MultiTableFailurePhase.CHECKPOINT, error));
         waitRuntimeTableFailuresHandled();
         subSinkErrorCheck();
         if (multiTableCommitInfo.getCommitInfo().isEmpty()) {
@@ -850,21 +872,24 @@ public class MultiTableSinkWriter
     /**
      * Closes this writer and releases all resources.
      *
-     * <p>Drains remaining queues, then calls {@link 
ExecutorService#shutdownNow()} to interrupt all
-     * consumer threads. Each sub-writer is closed under its respective {@link
-     * MultiTableWriterRunnable} lock to avoid concurrent access. Finally, the 
shared {@link
+     * <p>Signals any active timer flush round to stop, drains pending writes, 
and interrupts the
+     * executor threads. After any in-progress table isolation finishes, each 
sub-writer is closed
+     * under its queue lock, which also protects any in-flight flush. Finally, 
the shared {@link
      * MultiTableResourceManager} is closed and this method waits for the 
executor threads to stop.
      *
      * <p>Uses first-exception-wins error handling: if multiple sub-writers 
throw during close, only
      * the first exception is propagated. Resource manager close errors are 
logged but not
      * propagated.
      *
-     * @throws RuntimeException wrapping the first {@link Throwable} caught 
from any sub-writer;
-     *     note that the method signature declares {@code throws IOException} 
(inherited from the
-     *     interface), but the current implementation always wraps in {@code 
RuntimeException}
+     * @throws IOException if shutdown reports failed tables for a batch job
+     * @throws RuntimeException if shutdown fails without a batch failed-table 
report
      */
     @Override
     public void close() throws IOException {
+        CompletableFuture<Throwable> flush = 
activeTimerFlush.getAndSet(TIMER_FLUSH_CLOSED);
+        if (flush != null && flush != TIMER_FLUSH_CLOSED) {
+            flush.complete(new IOException("Multi-table sink writer is 
closing"));
+        }
         // The variables used in lambda expressions should be final or valid 
final, so they are
         // modified to arrays
         final Throwable[] firstE = {null};
@@ -875,6 +900,21 @@ public class MultiTableSinkWriter
             firstE[0] = e;
         }
         executorService.shutdownNow();
+        if (failurePolicy.continueOtherTables()) {
+            synchronized (this) {
+                // When a timer flush round is cancelled, a worker may already 
be isolating
+                // a failed table. That isolation can continue after 
shutdownNow() returns
+                // while the worker closes and removes the table's writers 
across queues.
+                //
+                // Queue locks serialize individual close calls. If shutdown 
closes a writer
+                // first, the writer is still in the map and isolation could 
close it again.
+                //
+                // handleTableFailure() holds this monitor. A flush worker 
checks the round's
+                // stop signal under the same monitor before starting 
isolation. Since close()
+                // has already signaled the round to stop, acquiring this 
monitor waits for any
+                // isolation in progress. Release it before acquiring the 
queue locks below.
+            }
+        }
         for (int i = 0; i < sinkWritersWithIndex.size(); i++) {
             synchronized (runnable.get(i)) {
                 Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> 
sinkIdentifierSinkWriterMap =
@@ -947,37 +987,130 @@ public class MultiTableSinkWriter
     }
 
     /**
-     * Aggregated flush triggered by the engine timer. Drains all blocking 
queues first, then calls
-     * each sub-writer's flush action under the corresponding {@link 
MultiTableWriterRunnable} lock
-     * to prevent concurrent writes.
+     * Drains the row queues and flushes independent queues in parallel. A 
successful round waits
+     * for every queue. A fatal failure ends the round and prevents further 
child flush actions.
      *
      * @param proxyContexts map from sink identifier to its {@link 
SinkContextProxy}
      */
     void aggregatedFlush(Map<SinkIdentifier, SinkContextProxy> proxyContexts) 
throws Exception {
         checkQueueRemain();
         subSinkErrorCheck();
-        for (int i = 0; i < sinkWritersWithIndex.size(); i++) {
-            synchronized (runnable.get(i)) {
-                for (SinkIdentifier id : sinkWritersWithIndex.get(i).keySet()) 
{
+        // A null result means all queues succeeded. Any completed result 
stops new actions.
+        CompletableFuture<Throwable> completion = new CompletableFuture<>();
+        if (!activeTimerFlush.compareAndSet(null, completion)) {
+            throw new IOException(
+                    "Multi-table timer flush is already running or the writer 
is closing");
+        }
+        List<Future<?>> flushTasks = new ArrayList<>();
+        boolean succeeded = false;
+        try {
+            if (sinkWritersWithIndex.size() == 1) {
+                flushQueue(0, proxyContexts, completion);
+                completion.complete(null);
+            } else {
+                try {
+                    submitQueueFlushes(proxyContexts, completion, flushTasks);
+                } catch (RejectedExecutionException error) {
+                    // Closing can shut down the executor after the submit 
loop checks completion.
+                    completion.complete(error);
+                }
+            }
+            Throwable failure = completion.get();
+            if (failure instanceof Exception) {
+                throw (Exception) failure;
+            }
+            if (failure instanceof Error) {
+                throw (Error) failure;
+            }
+            if (failure != null) {
+                throw new ExecutionException(failure);
+            }
+            succeeded = true;
+        } catch (InterruptedException error) {
+            Thread.currentThread().interrupt();
+            throw error;
+        } finally {
+            if (!succeeded) {
+                completion.cancel(false);
+                flushTasks.forEach(task -> task.cancel(true));
+            }
+            activeTimerFlush.compareAndSet(completion, null);
+        }
+    }
+
+    private void submitQueueFlushes(
+            Map<SinkIdentifier, SinkContextProxy> proxyContexts,
+            CompletableFuture<Throwable> completion,
+            List<Future<?>> flushTasks) {
+        int queueCount = sinkWritersWithIndex.size();
+        AtomicInteger remainingQueues = new AtomicInteger(queueCount);
+        for (int i = 0; i < queueCount && !completion.isDone(); i++) {
+            int queueIndex = i;
+            flushTasks.add(
+                    executorService.submit(
+                            () -> {
+                                try {
+                                    flushQueue(queueIndex, proxyContexts, 
completion);
+                                    if (remainingQueues.decrementAndGet() == 
0) {
+                                        completion.complete(null);
+                                    }
+                                } catch (Throwable error) {
+                                    if (!completion.complete(error)) {
+                                        log.warn(
+                                                "Timer flush task for queue {} 
failed after the round ended",
+                                                queueIndex,
+                                                error);
+                                    }
+                                }
+                            }));
+        }
+    }
+
+    private void flushQueue(
+            int queueIndex,
+            Map<SinkIdentifier, SinkContextProxy> proxyContexts,
+            CompletableFuture<Throwable> completion) {
+        MultiTableWriterRunnable writerRunnable = runnable.get(queueIndex);
+        for (SinkIdentifier id : 
sinkWritersWithIndex.get(queueIndex).keySet()) {
+            try {
+                synchronized (writerRunnable) {
+                    if (completion.isDone()) {
+                        return;
+                    }
+                    if (isTableFailed(id.getTableIdentifier())) {
+                        continue;
+                    }
                     SinkContextProxy proxy = proxyContexts.get(id);
-                    if (proxy != null && proxy.getFlushAction() != null) {
-                        try {
-                            proxy.getFlushAction().run();
-                        } catch (InterruptedException error) {
-                            Thread.currentThread().interrupt();
-                            throw error;
-                        } catch (Exception error) {
-                            if (failurePolicy.continueOtherTables()) {
-                                handleTableFailure(
-                                        id.getTableIdentifier(),
-                                        MultiTableFailurePhase.TIMER_FLUSH,
-                                        error);
-                                continue;
-                            }
-                            throw error;
+                    RunnableWithException flushAction =
+                            proxy == null ? null : proxy.getFlushAction();
+                    if (flushAction != null) {
+                        flushAction.run();
+                    }
+                }
+            } catch (Throwable error) {
+                if (error instanceof InterruptedException) {
+                    Thread.currentThread().interrupt();
+                } else if (error instanceof Exception && 
failurePolicy.continueOtherTables()) {
+                    // Table isolation acquires other queue locks. Coordinate 
with close outside
+                    // ours.
+                    synchronized (this) {
+                        if (!completion.isDone()) {
+                            handleTableFailure(
+                                    id.getTableIdentifier(),
+                                    MultiTableFailurePhase.TIMER_FLUSH,
+                                    error);
+                            continue;
                         }
                     }
                 }
+                if (!completion.complete(error)) {
+                    log.warn(
+                            "Timer flush failed for table {} in queue {} after 
the round ended",
+                            id.getTableIdentifier(),
+                            queueIndex,
+                            error);
+                }
+                return;
             }
         }
     }
diff --git 
a/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java
 
b/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java
index 6035c52b4d..4e6c3c4acb 100644
--- 
a/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java
+++ 
b/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java
@@ -47,6 +47,7 @@ import lombok.Data;
 import java.io.IOException;
 import java.io.Serializable;
 import java.lang.reflect.Field;
+import java.time.Duration;
 import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
@@ -54,11 +55,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -488,6 +492,61 @@ public class MultiTableSinkWriterTest {
         Assertions.assertDoesNotThrow(multiTableSinkWriter::close);
     }
 
+    @Test
+    public void testPrepareCommitIsolatesConcurrentFailures() {
+        Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new 
HashMap<>();
+        Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new 
HashMap<>();
+        SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy", 
2);
+        sinkWriters.put(healthyIdentifier, new TestSinkWriter());
+        sinkWritersContext.put(healthyIdentifier, new TestSinkWriterContext());
+
+        CountDownLatch preparesStarted = new CountDownLatch(2);
+        AtomicInteger failedPrepares = new AtomicInteger();
+        for (int queueIndex = 0; queueIndex < 2; queueIndex++) {
+            SinkIdentifier failedIdentifier =
+                    SinkIdentifier.of("test.failed" + queueIndex, queueIndex);
+            sinkWriters.put(
+                    failedIdentifier,
+                    new TestSinkWriter() {
+                        @Override
+                        public Optional<TestSinkState> prepareCommit(long 
checkpointId)
+                                throws IOException {
+                            preparesStarted.countDown();
+                            try {
+                                Assertions.assertTrue(preparesStarted.await(5, 
TimeUnit.SECONDS));
+                            } catch (InterruptedException e) {
+                                Thread.currentThread().interrupt();
+                                throw new IOException(e);
+                            }
+                            failedPrepares.incrementAndGet();
+                            throw new IOException("intentional prepare 
failure");
+                        }
+                    });
+            sinkWritersContext.put(failedIdentifier, new 
TestSinkWriterContext());
+        }
+
+        MultiTableSinkWriter multiTableSinkWriter =
+                new MultiTableSinkWriter(
+                        sinkWriters,
+                        2,
+                        sinkWritersContext,
+                        MultiTableFailurePolicy.CONTINUE_OTHER_TABLES,
+                        JobMode.BATCH);
+        MultiTableCommitInfo commitInfo =
+                Assertions.assertTimeoutPreemptively(
+                        Duration.ofSeconds(10),
+                        () -> 
multiTableSinkWriter.prepareCommit(1L).orElseThrow());
+        Assertions.assertEquals(2, failedPrepares.get());
+        Assertions.assertEquals(
+                Collections.singleton(healthyIdentifier), 
commitInfo.getCommitInfo().keySet());
+
+        IOException closeException =
+                Assertions.assertThrows(IOException.class, 
multiTableSinkWriter::close);
+        
Assertions.assertTrue(closeException.getMessage().contains("test.failed0"));
+        
Assertions.assertTrue(closeException.getMessage().contains("test.failed1"));
+        
Assertions.assertTrue(closeException.getMessage().contains("phase=checkpoint"));
+    }
+
     @Test
     public void testPrepareCommitFatalRowErrorHandlingFailureFailsFast() {
         Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new 
HashMap<>();
@@ -570,28 +629,35 @@ public class MultiTableSinkWriterTest {
     }
 
     @Test
-    public void testAggregatedFlushIsolatesFailedTableAndFlushesHealthyTable() 
{
+    public void 
testAggregatedFlushIsolatesConcurrentFailuresAndFlushesHealthyTable() {
         Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new 
HashMap<>();
         Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new 
HashMap<>();
         Map<SinkIdentifier, SinkContextProxy> proxyContexts = new HashMap<>();
-        SinkIdentifier failedIdentifier = SinkIdentifier.of("test.failed", 0);
-        SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy", 
1);
-        sinkWriters.put(failedIdentifier, new TestSinkWriter());
+        SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy", 
2);
         sinkWriters.put(healthyIdentifier, new TestSinkWriter());
-        sinkWritersContext.put(failedIdentifier, new TestSinkWriterContext());
         sinkWritersContext.put(healthyIdentifier, new TestSinkWriterContext());
 
         AtomicInteger failedFlushCount = new AtomicInteger();
-        SinkContextProxy failedProxy = new SinkContextProxy(0, 2, new 
TestSinkWriterContext());
-        failedProxy.registerFlushAction(
-                () -> {
-                    failedFlushCount.incrementAndGet();
-                    throw new IOException("intentional flush failure");
-                });
+        CountDownLatch failedFlushesStarted = new CountDownLatch(2);
+        for (int queueIndex = 0; queueIndex < 2; queueIndex++) {
+            SinkIdentifier failedIdentifier =
+                    SinkIdentifier.of("test.failed" + queueIndex, queueIndex);
+            sinkWriters.put(failedIdentifier, new TestSinkWriter());
+            sinkWritersContext.put(failedIdentifier, new 
TestSinkWriterContext());
+            SinkContextProxy failedProxy =
+                    new SinkContextProxy(queueIndex, 2, new 
TestSinkWriterContext());
+            failedProxy.registerFlushAction(
+                    () -> {
+                        failedFlushCount.incrementAndGet();
+                        failedFlushesStarted.countDown();
+                        Assertions.assertTrue(failedFlushesStarted.await(5, 
TimeUnit.SECONDS));
+                        throw new IOException("intentional flush failure");
+                    });
+            proxyContexts.put(failedIdentifier, failedProxy);
+        }
         AtomicInteger healthyFlushCount = new AtomicInteger();
-        SinkContextProxy healthyProxy = new SinkContextProxy(1, 2, new 
TestSinkWriterContext());
+        SinkContextProxy healthyProxy = new SinkContextProxy(2, 2, new 
TestSinkWriterContext());
         healthyProxy.registerFlushAction(healthyFlushCount::incrementAndGet);
-        proxyContexts.put(failedIdentifier, failedProxy);
         proxyContexts.put(healthyIdentifier, healthyProxy);
 
         MultiTableSinkWriter multiTableSinkWriter =
@@ -602,61 +668,178 @@ public class MultiTableSinkWriterTest {
                         MultiTableFailurePolicy.CONTINUE_OTHER_TABLES,
                         JobMode.BATCH);
 
-        Assertions.assertDoesNotThrow(() -> 
multiTableSinkWriter.aggregatedFlush(proxyContexts));
-        Assertions.assertEquals(1, failedFlushCount.get());
+        Assertions.assertTimeoutPreemptively(
+                Duration.ofSeconds(10), () -> 
multiTableSinkWriter.aggregatedFlush(proxyContexts));
+        Assertions.assertEquals(2, failedFlushCount.get());
         Assertions.assertEquals(1, healthyFlushCount.get());
 
         Assertions.assertDoesNotThrow(() -> 
multiTableSinkWriter.aggregatedFlush(proxyContexts));
-        Assertions.assertEquals(1, failedFlushCount.get());
+        Assertions.assertEquals(2, failedFlushCount.get());
         Assertions.assertEquals(2, healthyFlushCount.get());
 
         IOException closeException =
                 Assertions.assertThrows(IOException.class, 
multiTableSinkWriter::close);
-        
Assertions.assertTrue(closeException.getMessage().contains("test.failed"));
+        
Assertions.assertTrue(closeException.getMessage().contains("test.failed0"));
+        
Assertions.assertTrue(closeException.getMessage().contains("test.failed1"));
         
Assertions.assertTrue(closeException.getMessage().contains("phase=timer_flush"));
     }
 
     @Test
-    public void testAggregatedFlushFailFastStopsAtFirstFailure() {
+    public void 
testAggregatedFlushReportsFailureAndClosesQueuesIndependently() throws 
Exception {
         Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new 
HashMap<>();
         Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new 
HashMap<>();
         Map<SinkIdentifier, SinkContextProxy> proxyContexts = new HashMap<>();
         SinkIdentifier failedIdentifier = SinkIdentifier.of("test.failed", 0);
-        SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy", 
1);
-        sinkWriters.put(failedIdentifier, new TestSinkWriter());
-        sinkWriters.put(healthyIdentifier, new TestSinkWriter());
+        SinkIdentifier slowIdentifier = SinkIdentifier.of("test.slow", 1);
+        SinkIdentifier siblingIdentifier = SinkIdentifier.of("test.sibling", 
3);
+        CountDownLatch failedWriterClosed = new CountDownLatch(1);
+        AtomicBoolean slowFlushFinished = new AtomicBoolean();
+        AtomicBoolean slowWriterClosed = new AtomicBoolean();
+        sinkWriters.put(
+                failedIdentifier,
+                new TestSinkWriter() {
+                    @Override
+                    public void close() {
+                        failedWriterClosed.countDown();
+                    }
+                });
+        sinkWriters.put(
+                slowIdentifier,
+                new TestSinkWriter() {
+                    @Override
+                    public void close() {
+                        Assertions.assertTrue(slowFlushFinished.get());
+                        slowWriterClosed.set(true);
+                    }
+                });
+        sinkWriters.put(siblingIdentifier, new TestSinkWriter());
+        sinkWritersContext.put(slowIdentifier, new TestSinkWriterContext());
+        sinkWritersContext.put(siblingIdentifier, new TestSinkWriterContext());
         sinkWritersContext.put(failedIdentifier, new TestSinkWriterContext());
-        sinkWritersContext.put(healthyIdentifier, new TestSinkWriterContext());
 
-        AtomicInteger failedFlushCount = new AtomicInteger();
+        AtomicInteger slowQueueFlushCount = new AtomicInteger();
+        CountDownLatch slowFlushStarted = new CountDownLatch(1);
+        CompletableFuture<Void> releaseSlowFlush = new CompletableFuture<>();
+        IOException expectedFailure = new IOException("intentional flush 
failure");
+        SinkContextProxy slowProxy = new SinkContextProxy(1, 2, new 
TestSinkWriterContext());
+        slowProxy.registerFlushAction(
+                () -> {
+                    // Block whichever child is visited first on this queue.
+                    if (slowQueueFlushCount.incrementAndGet() == 1) {
+                        slowFlushStarted.countDown();
+                        releaseSlowFlush.join();
+                        slowFlushFinished.set(true);
+                    }
+                });
+        SinkContextProxy siblingProxy = new SinkContextProxy(3, 2, new 
TestSinkWriterContext());
+        siblingProxy.registerFlushAction(slowProxy.getFlushAction());
         SinkContextProxy failedProxy = new SinkContextProxy(0, 2, new 
TestSinkWriterContext());
         failedProxy.registerFlushAction(
                 () -> {
-                    failedFlushCount.incrementAndGet();
-                    throw new IOException("intentional flush failure");
+                    Assertions.assertTrue(slowFlushStarted.await(5, 
TimeUnit.SECONDS));
+                    throw expectedFailure;
                 });
-        AtomicInteger healthyFlushCount = new AtomicInteger();
-        SinkContextProxy healthyProxy = new SinkContextProxy(1, 2, new 
TestSinkWriterContext());
-        healthyProxy.registerFlushAction(healthyFlushCount::incrementAndGet);
+        proxyContexts.put(slowIdentifier, slowProxy);
+        proxyContexts.put(siblingIdentifier, siblingProxy);
         proxyContexts.put(failedIdentifier, failedProxy);
-        proxyContexts.put(healthyIdentifier, healthyProxy);
 
         MultiTableSinkWriter multiTableSinkWriter =
-                new MultiTableSinkWriter(
-                        sinkWriters,
-                        2,
-                        sinkWritersContext,
-                        MultiTableFailurePolicy.FAIL_FAST,
-                        JobMode.BATCH);
+                new MultiTableSinkWriter(sinkWriters, 2, sinkWritersContext);
+        FutureTask<Void> flush =
+                new FutureTask<>(
+                        () -> {
+                            
multiTableSinkWriter.aggregatedFlush(proxyContexts);
+                            return null;
+                        });
+        Thread dispatcher = new Thread(flush);
+        dispatcher.setDaemon(true);
+        FutureTask<Void> close =
+                new FutureTask<>(
+                        () -> {
+                            multiTableSinkWriter.close();
+                            return null;
+                        });
+        Thread closer = new Thread(close);
+        closer.setDaemon(true);
+        try {
+            multiTableSinkWriter.write(buildRow("test.failed", 0));
+            dispatcher.start();
+            ExecutionException error =
+                    Assertions.assertThrows(
+                            ExecutionException.class, () -> flush.get(5, 
TimeUnit.SECONDS));
+            Assertions.assertSame(expectedFailure, error.getCause());
 
-        IOException flushException =
-                Assertions.assertThrows(
-                        IOException.class,
-                        () -> 
multiTableSinkWriter.aggregatedFlush(proxyContexts));
-        Assertions.assertEquals("intentional flush failure", 
flushException.getMessage());
-        Assertions.assertEquals(1, failedFlushCount.get());
-        Assertions.assertEquals(0, healthyFlushCount.get());
-        Assertions.assertDoesNotThrow(multiTableSinkWriter::close);
+            closer.start();
+            Assertions.assertTrue(failedWriterClosed.await(5, 
TimeUnit.SECONDS));
+            Assertions.assertFalse(slowWriterClosed.get());
+        } finally {
+            releaseSlowFlush.complete(null);
+            dispatcher.join(5000);
+            if (closer.getState() == Thread.State.NEW) {
+                closer.start();
+            }
+            close.get(5, TimeUnit.SECONDS);
+        }
+        Assertions.assertTrue(slowWriterClosed.get());
+        Assertions.assertEquals(1, slowQueueFlushCount.get());
+    }
+
+    @Test
+    public void testInterruptedAggregatedFlushDoesNotStartAnotherChildFlush() 
throws Exception {
+        Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new 
HashMap<>();
+        Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new 
HashMap<>();
+        Map<SinkIdentifier, SinkContextProxy> proxyContexts = new HashMap<>();
+        AtomicInteger startedFlushes = new AtomicInteger();
+        CountDownLatch firstFlushStarted = new CountDownLatch(1);
+        CompletableFuture<Void> releaseFirstFlush = new CompletableFuture<>();
+        for (int index : new int[] {0, 2}) {
+            SinkIdentifier identifier = SinkIdentifier.of("test.table" + 
index, index);
+            SinkWriter.Context context = new TestSinkWriterContext();
+            sinkWriters.put(identifier, new TestSinkWriter());
+            sinkWritersContext.put(identifier, context);
+            SinkContextProxy proxy = new SinkContextProxy(index, 2, context);
+            proxy.registerFlushAction(
+                    () -> {
+                        if (startedFlushes.incrementAndGet() == 1) {
+                            firstFlushStarted.countDown();
+                            releaseFirstFlush.join();
+                            // Simulate a connector that consumes the 
interrupt during I/O.
+                            Thread.interrupted();
+                        }
+                    });
+            proxyContexts.put(identifier, proxy);
+        }
+
+        MultiTableSinkWriter multiTableSinkWriter =
+                new MultiTableSinkWriter(sinkWriters, 2, sinkWritersContext);
+        AtomicBoolean dispatcherInterrupted = new AtomicBoolean();
+        FutureTask<Void> flush =
+                new FutureTask<>(
+                        () -> {
+                            try {
+                                
multiTableSinkWriter.aggregatedFlush(proxyContexts);
+                                return null;
+                            } finally {
+                                
dispatcherInterrupted.set(Thread.currentThread().isInterrupted());
+                            }
+                        });
+        Thread dispatcher = new Thread(flush);
+        dispatcher.setDaemon(true);
+        dispatcher.start();
+        try {
+            Assertions.assertTrue(firstFlushStarted.await(5, 
TimeUnit.SECONDS));
+            dispatcher.interrupt();
+            ExecutionException error =
+                    Assertions.assertThrows(
+                            ExecutionException.class, () -> flush.get(5, 
TimeUnit.SECONDS));
+            Assertions.assertTrue(error.getCause() instanceof 
InterruptedException);
+            Assertions.assertTrue(dispatcherInterrupted.get());
+        } finally {
+            releaseFirstFlush.complete(null);
+            dispatcher.join(5000);
+            multiTableSinkWriter.close();
+        }
+        Assertions.assertEquals(1, startedFlushes.get());
     }
 
     @Test

Reply via email to