This is an automated email from the ASF dual-hosted git repository.
nzw921rx pushed a commit to branch 3.1.0-release
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/3.1.0-release by this push:
new 901381fc7f [Improve][API] Run multi-table timer flushes in parallel
(#13201)
901381fc7f is described below
commit 901381fc7fe3bb12d1d9efc166d6d4f4da1c24ea
Author: zhiweiniu <[email protected]>
AuthorDate: Thu Nov 12 16:04:57 2026 +0800
[Improve][API] Run multi-table timer flushes in parallel (#13201)
(cherry picked from commit 466d4168a50e4b1bfd1f8ad8ae9eebaf50d509a6)
---
docs/en/engines/zeta/timer-flush.md | 10 +-
docs/zh/engines/zeta/timer-flush.md | 9 +-
.../api/sink/multitablesink/MultiTableSink.java | 5 +-
.../sink/multitablesink/MultiTableSinkWriter.java | 233 ++++++++++++++----
.../multitablesink/MultiTableSinkWriterTest.java | 269 +++++++++++++++++----
5 files changed, 425 insertions(+), 101 deletions(-)
diff --git a/docs/en/engines/zeta/timer-flush.md
b/docs/en/engines/zeta/timer-flush.md
index 0ba3bef4a7..d710096c9e 100644
--- a/docs/en/engines/zeta/timer-flush.md
+++ b/docs/en/engines/zeta/timer-flush.md
@@ -179,11 +179,15 @@ responsible for any buffered data that has not been
flushed by a timer.
For a Multi-Table Sink, the Sink registers one aggregated action when at least
one child writer
supports timer flush. Before invoking child actions, the Multi-Table writer
waits for its internal
row queues and in-progress writes to finish. This prevents timer flush from
racing with its
-asynchronous writer threads.
+asynchronous writer threads. After the drain, child writers flush in parallel
across queues and
+sequentially within each queue. On success, the Sink waits for every queue
before processing later
+records.
The configured Multi-Table failure policy also applies to timer flush.
`FAIL_FAST` propagates the
-first child failure. A policy that continues other tables isolates the failed
table and continues
-flushing healthy tables.
+first reported child failure promptly. A failure that ends the round or a
cancellation stops
+the queues from starting further child flush actions. During shutdown, each
queue's writers close
+after its in-flight flush exits. Shared resources are released after all
writers close. A policy
+that continues other tables isolates the failed table and continues flushing
healthy tables.
## Monitoring
diff --git a/docs/zh/engines/zeta/timer-flush.md
b/docs/zh/engines/zeta/timer-flush.md
index ef3686c179..b738dbc3db 100644
--- a/docs/zh/engines/zeta/timer-flush.md
+++ b/docs/zh/engines/zeta/timer-flush.md
@@ -161,10 +161,13 @@ Connector 提供的幂等能力或确定性键。
对于 Multi-Table Sink,只要至少一个子 Writer 支持定时刷新,Sink 就会注册一个聚合操作。
调用子 Writer 操作前,Multi-Table Writer 会等待内部数据队列和正在执行的写入完成,避免定时
-刷新与异步写线程发生竞争。
+刷新与异步写线程发生竞争。排空后,不同队列并行刷新,同一队列内的子 Writer 依次刷新。
+刷新成功时,Sink 等待各队列结束后再处理后续记录。
-定时刷新也遵循已配置的 Multi-Table 失败策略。`FAIL_FAST` 会传递第一个子 Writer 异常;
-允许继续处理其他表的策略会隔离失败表,并继续刷新健康表。
+定时刷新也遵循已配置的 Multi-Table 失败策略。`FAIL_FAST` 收到首个子 Writer 异常后立即传递。
+刷新因失败或取消而结束后,各队列停止启动新的子 Writer 刷新。Sink 关闭时,每个队列会等本队列
+正在执行的刷新退出后再关闭 Writer。所有 Writer 关闭后才释放共享资源。允许继续处理其他表的策略
+会隔离失败表,并继续刷新健康表。
## 监控指标
diff --git
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java
index 34bf0dc07f..36e8cd52c4 100644
---
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java
+++
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSink.java
@@ -259,8 +259,9 @@ public class MultiTableSink
* Registers an aggregated flush action on the parent context if any
sub-writer registered a
* flush action via its {@link SinkContextProxy}.
*
- * <p>The registered action drains all blocking queues and then calls each
sub-writer's flush
- * action under the corresponding lock, ensuring safe execution from the
engine timer thread.
+ * <p>The sink task invokes the action when it consumes a flush signal.
The action drains
+ * pending writes and flushes independent queues in parallel, with child
writers serialized
+ * under their queue lock.
*/
private void registerAggregatedFlushIfNeeded(
SinkWriter.Context context,
diff --git
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
index 6037894acd..271f6df30e 100644
---
a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
+++
b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriter.java
@@ -31,6 +31,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.tracing.MDCTracer;
import org.apache.seatunnel.common.constants.JobMode;
import org.apache.seatunnel.common.utils.HashUtils;
+import org.apache.seatunnel.common.utils.function.RunnableWithException;
import lombok.extern.slf4j.Slf4j;
@@ -46,14 +47,18 @@ import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -66,10 +71,9 @@ import java.util.stream.Collectors;
* appropriate sub-writer. Routing is based on the row's primary key hash for
ordered delivery, or
* random selection when no primary key exists.
*
- * <p>Thread pool sizing is {@code queueSize * 2}: half for queue consumer
threads, half for
- * parallel {@link #prepareCommit(long)} tasks. Synchronization between the
consumer threads and
- * checkpoint/commit operations is coordinated through locks on each {@link
- * MultiTableWriterRunnable} instance.
+ * <p>Thread pool sizing is {@code queueSize * 2}, with half for queue
consumer threads and half for
+ * parallel timer flush and {@link #prepareCommit(long)} tasks. These
operations synchronize with
+ * the consumer threads through locks on each {@link MultiTableWriterRunnable}
instance.
*/
@Slf4j
public class MultiTableSinkWriter
@@ -77,6 +81,8 @@ public class MultiTableSinkWriter
SupportSchemaEvolutionSinkWriter {
private static final long EXECUTOR_CLOSE_TIMEOUT_SECONDS = 60L;
+ private static final CompletableFuture<Throwable> TIMER_FLUSH_CLOSED =
+ CompletableFuture.completedFuture(null);
private final Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>>
sinkWriters;
private final Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext;
@@ -89,6 +95,8 @@ public class MultiTableSinkWriter
private final List<BlockingQueue<MultiTableWriterRunnable.QueueElement>>
blockingQueues =
new ArrayList<>();
private final ExecutorService executorService;
+ private final AtomicReference<CompletableFuture<Throwable>>
activeTimerFlush =
+ new AtomicReference<>();
private final MultiTableFailurePolicy failurePolicy;
private final JobMode jobMode;
private final int tableRetryTimes;
@@ -108,9 +116,8 @@ public class MultiTableSinkWriter
*
* <p>Initializes a fixed thread pool of size {@code queueSize * 2}: half
the threads run {@link
* MultiTableWriterRunnable} consumers that drain the blocking queues, and
the other half are
- * reserved for parallel {@link #prepareCommit(long)} tasks. Each
sub-writer is assigned to a
- * queue using the formula {@code sinkIdentifier.getIndex() % queueSize},
ensuring deterministic
- * distribution across queues.
+ * reserved for timer flush and parallel {@link #prepareCommit(long)}
tasks. Each sub-writer is
+ * assigned to a queue using the formula {@code sinkIdentifier.getIndex()
% queueSize}.
*
* @param sinkWriters map of sink identifiers to their corresponding
writers
* @param queueSize number of blocking queues (and consumer threads) to
create
@@ -204,9 +211,7 @@ public class MultiTableSinkWriter
executorService =
MDCTracer.tracing(
Executors.newFixedThreadPool(
- // we use it in `MultiTableWriterRunnable` and
`prepare commit
- // task`, so it
- // should be double.
+ // Reserve one thread per queue for flush and
prepare commit tasks.
queueSize * 2,
runnable -> {
Thread thread = new Thread(runnable);
@@ -746,12 +751,15 @@ public class MultiTableSinkWriter
subSinkErrorCheck();
MultiTableCommitInfo multiTableCommitInfo =
new MultiTableCommitInfo(new ConcurrentHashMap<>());
+ ConcurrentMap<String, Throwable> checkpointFailures = new
ConcurrentHashMap<>();
List<Future<?>> futures = new ArrayList<>();
for (int i = 0; i < sinkWritersWithIndex.size(); i++) {
int subWriterIndex = i;
futures.add(
executorService.submit(
() -> {
+ String fatalFailureTable = null;
+ Throwable fatalFailure = null;
synchronized (runnable.get(subWriterIndex)) {
for (Map.Entry<SinkIdentifier,
SinkWriter<SeaTunnelRow, ?, ?>>
sinkWriterEntry :
@@ -759,15 +767,19 @@ public class MultiTableSinkWriter
sinkWritersWithIndex
.get(subWriterIndex)
.entrySet())) {
+ SinkIdentifier sinkIdentifier =
sinkWriterEntry.getKey();
+ String tableId =
sinkIdentifier.getTableIdentifier();
+ if (isTableFailed(tableId)
+ ||
checkpointFailures.containsKey(tableId)) {
+ continue;
+ }
Optional<?> commit;
try {
SinkWriter<SeaTunnelRow, ?, ?>
sinkWriter =
sinkWriterEntry.getValue();
commit =
executeWithTableRetry(
- sinkWriterEntry
- .getKey()
-
.getTableIdentifier(),
+ tableId,
MultiTableFailurePhase.CHECKPOINT,
() ->
sinkWriter.prepareCommit(
@@ -776,24 +788,30 @@ public class MultiTableSinkWriter
Thread.currentThread().interrupt();
throw new RuntimeException(error);
} catch (Throwable error) {
- if
(failurePolicy.continueOtherTables()) {
- handleTableFailure(
- sinkWriterEntry
- .getKey()
-
.getTableIdentifier(),
-
MultiTableFailurePhase.CHECKPOINT,
- error);
- continue;
+ if
(!failurePolicy.continueOtherTables()) {
+ throw new
RuntimeException(error);
}
- throw new RuntimeException(error);
+ if
(containsFatalRowErrorHandlingFailure(error)) {
+ fatalFailureTable = tableId;
+ fatalFailure = error;
+ break;
+ }
+
checkpointFailures.putIfAbsent(tableId, error);
+ continue;
}
commit.ifPresent(
o ->
multiTableCommitInfo
.getCommitInfo()
-
.put(sinkWriterEntry.getKey(), o));
+
.put(sinkIdentifier, o));
}
}
+ if (fatalFailure != null) {
+ handleTableFailure(
+ fatalFailureTable,
+ MultiTableFailurePhase.CHECKPOINT,
+ fatalFailure);
+ }
}));
}
for (Future<?> future : futures) {
@@ -803,6 +821,10 @@ public class MultiTableSinkWriter
throw new RuntimeException(e);
}
}
+ // Table isolation acquires other queue locks. Run it after the
prepare tasks finish.
+ checkpointFailures.forEach(
+ (tableId, error) ->
+ handleTableFailure(tableId,
MultiTableFailurePhase.CHECKPOINT, error));
waitRuntimeTableFailuresHandled();
subSinkErrorCheck();
if (multiTableCommitInfo.getCommitInfo().isEmpty()) {
@@ -850,21 +872,24 @@ public class MultiTableSinkWriter
/**
* Closes this writer and releases all resources.
*
- * <p>Drains remaining queues, then calls {@link
ExecutorService#shutdownNow()} to interrupt all
- * consumer threads. Each sub-writer is closed under its respective {@link
- * MultiTableWriterRunnable} lock to avoid concurrent access. Finally, the
shared {@link
+ * <p>Signals any active timer flush round to stop, drains pending writes,
and interrupts the
+ * executor threads. After any in-progress table isolation finishes, each
sub-writer is closed
+ * under its queue lock, which also protects any in-flight flush. Finally,
the shared {@link
* MultiTableResourceManager} is closed and this method waits for the
executor threads to stop.
*
* <p>Uses first-exception-wins error handling: if multiple sub-writers
throw during close, only
* the first exception is propagated. Resource manager close errors are
logged but not
* propagated.
*
- * @throws RuntimeException wrapping the first {@link Throwable} caught
from any sub-writer;
- * note that the method signature declares {@code throws IOException}
(inherited from the
- * interface), but the current implementation always wraps in {@code
RuntimeException}
+ * @throws IOException if shutdown reports failed tables for a batch job
+ * @throws RuntimeException if shutdown fails without a batch failed-table
report
*/
@Override
public void close() throws IOException {
+ CompletableFuture<Throwable> flush =
activeTimerFlush.getAndSet(TIMER_FLUSH_CLOSED);
+ if (flush != null && flush != TIMER_FLUSH_CLOSED) {
+ flush.complete(new IOException("Multi-table sink writer is
closing"));
+ }
// The variables used in lambda expressions should be final or valid
final, so they are
// modified to arrays
final Throwable[] firstE = {null};
@@ -875,6 +900,21 @@ public class MultiTableSinkWriter
firstE[0] = e;
}
executorService.shutdownNow();
+ if (failurePolicy.continueOtherTables()) {
+ synchronized (this) {
+ // When a timer flush round is cancelled, a worker may already
be isolating
+ // a failed table. That isolation can continue after
shutdownNow() returns
+ // while the worker closes and removes the table's writers
across queues.
+ //
+ // Queue locks serialize individual close calls. If shutdown
closes a writer
+ // first, the writer is still in the map and isolation could
close it again.
+ //
+ // handleTableFailure() holds this monitor. A flush worker
checks the round's
+ // stop signal under the same monitor before starting
isolation. Since close()
+ // has already signaled the round to stop, acquiring this
monitor waits for any
+ // isolation in progress. Release it before acquiring the
queue locks below.
+ }
+ }
for (int i = 0; i < sinkWritersWithIndex.size(); i++) {
synchronized (runnable.get(i)) {
Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>>
sinkIdentifierSinkWriterMap =
@@ -947,37 +987,130 @@ public class MultiTableSinkWriter
}
/**
- * Aggregated flush triggered by the engine timer. Drains all blocking
queues first, then calls
- * each sub-writer's flush action under the corresponding {@link
MultiTableWriterRunnable} lock
- * to prevent concurrent writes.
+ * Drains the row queues and flushes independent queues in parallel. A
successful round waits
+ * for every queue. A fatal failure ends the round and prevents further
child flush actions.
*
* @param proxyContexts map from sink identifier to its {@link
SinkContextProxy}
*/
void aggregatedFlush(Map<SinkIdentifier, SinkContextProxy> proxyContexts)
throws Exception {
checkQueueRemain();
subSinkErrorCheck();
- for (int i = 0; i < sinkWritersWithIndex.size(); i++) {
- synchronized (runnable.get(i)) {
- for (SinkIdentifier id : sinkWritersWithIndex.get(i).keySet())
{
+ // A null result means all queues succeeded. Any completed result
stops new actions.
+ CompletableFuture<Throwable> completion = new CompletableFuture<>();
+ if (!activeTimerFlush.compareAndSet(null, completion)) {
+ throw new IOException(
+ "Multi-table timer flush is already running or the writer
is closing");
+ }
+ List<Future<?>> flushTasks = new ArrayList<>();
+ boolean succeeded = false;
+ try {
+ if (sinkWritersWithIndex.size() == 1) {
+ flushQueue(0, proxyContexts, completion);
+ completion.complete(null);
+ } else {
+ try {
+ submitQueueFlushes(proxyContexts, completion, flushTasks);
+ } catch (RejectedExecutionException error) {
+ // Closing can shut down the executor after the submit
loop checks completion.
+ completion.complete(error);
+ }
+ }
+ Throwable failure = completion.get();
+ if (failure instanceof Exception) {
+ throw (Exception) failure;
+ }
+ if (failure instanceof Error) {
+ throw (Error) failure;
+ }
+ if (failure != null) {
+ throw new ExecutionException(failure);
+ }
+ succeeded = true;
+ } catch (InterruptedException error) {
+ Thread.currentThread().interrupt();
+ throw error;
+ } finally {
+ if (!succeeded) {
+ completion.cancel(false);
+ flushTasks.forEach(task -> task.cancel(true));
+ }
+ activeTimerFlush.compareAndSet(completion, null);
+ }
+ }
+
+ private void submitQueueFlushes(
+ Map<SinkIdentifier, SinkContextProxy> proxyContexts,
+ CompletableFuture<Throwable> completion,
+ List<Future<?>> flushTasks) {
+ int queueCount = sinkWritersWithIndex.size();
+ AtomicInteger remainingQueues = new AtomicInteger(queueCount);
+ for (int i = 0; i < queueCount && !completion.isDone(); i++) {
+ int queueIndex = i;
+ flushTasks.add(
+ executorService.submit(
+ () -> {
+ try {
+ flushQueue(queueIndex, proxyContexts,
completion);
+ if (remainingQueues.decrementAndGet() ==
0) {
+ completion.complete(null);
+ }
+ } catch (Throwable error) {
+ if (!completion.complete(error)) {
+ log.warn(
+ "Timer flush task for queue {}
failed after the round ended",
+ queueIndex,
+ error);
+ }
+ }
+ }));
+ }
+ }
+
+ private void flushQueue(
+ int queueIndex,
+ Map<SinkIdentifier, SinkContextProxy> proxyContexts,
+ CompletableFuture<Throwable> completion) {
+ MultiTableWriterRunnable writerRunnable = runnable.get(queueIndex);
+ for (SinkIdentifier id :
sinkWritersWithIndex.get(queueIndex).keySet()) {
+ try {
+ synchronized (writerRunnable) {
+ if (completion.isDone()) {
+ return;
+ }
+ if (isTableFailed(id.getTableIdentifier())) {
+ continue;
+ }
SinkContextProxy proxy = proxyContexts.get(id);
- if (proxy != null && proxy.getFlushAction() != null) {
- try {
- proxy.getFlushAction().run();
- } catch (InterruptedException error) {
- Thread.currentThread().interrupt();
- throw error;
- } catch (Exception error) {
- if (failurePolicy.continueOtherTables()) {
- handleTableFailure(
- id.getTableIdentifier(),
- MultiTableFailurePhase.TIMER_FLUSH,
- error);
- continue;
- }
- throw error;
+ RunnableWithException flushAction =
+ proxy == null ? null : proxy.getFlushAction();
+ if (flushAction != null) {
+ flushAction.run();
+ }
+ }
+ } catch (Throwable error) {
+ if (error instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ } else if (error instanceof Exception &&
failurePolicy.continueOtherTables()) {
+ // Table isolation acquires other queue locks. Coordinate
with close outside
+ // ours.
+ synchronized (this) {
+ if (!completion.isDone()) {
+ handleTableFailure(
+ id.getTableIdentifier(),
+ MultiTableFailurePhase.TIMER_FLUSH,
+ error);
+ continue;
}
}
}
+ if (!completion.complete(error)) {
+ log.warn(
+ "Timer flush failed for table {} in queue {} after
the round ended",
+ id.getTableIdentifier(),
+ queueIndex,
+ error);
+ }
+ return;
}
}
}
diff --git
a/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java
b/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java
index 6035c52b4d..4e6c3c4acb 100644
---
a/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java
+++
b/seatunnel-api/src/test/java/org/apache/seatunnel/api/sink/multitablesink/MultiTableSinkWriterTest.java
@@ -47,6 +47,7 @@ import lombok.Data;
import java.io.IOException;
import java.io.Serializable;
import java.lang.reflect.Field;
+import java.time.Duration;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
@@ -54,11 +55,14 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.FutureTask;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -488,6 +492,61 @@ public class MultiTableSinkWriterTest {
Assertions.assertDoesNotThrow(multiTableSinkWriter::close);
}
+ @Test
+ public void testPrepareCommitIsolatesConcurrentFailures() {
+ Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new
HashMap<>();
+ Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new
HashMap<>();
+ SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy",
2);
+ sinkWriters.put(healthyIdentifier, new TestSinkWriter());
+ sinkWritersContext.put(healthyIdentifier, new TestSinkWriterContext());
+
+ CountDownLatch preparesStarted = new CountDownLatch(2);
+ AtomicInteger failedPrepares = new AtomicInteger();
+ for (int queueIndex = 0; queueIndex < 2; queueIndex++) {
+ SinkIdentifier failedIdentifier =
+ SinkIdentifier.of("test.failed" + queueIndex, queueIndex);
+ sinkWriters.put(
+ failedIdentifier,
+ new TestSinkWriter() {
+ @Override
+ public Optional<TestSinkState> prepareCommit(long
checkpointId)
+ throws IOException {
+ preparesStarted.countDown();
+ try {
+ Assertions.assertTrue(preparesStarted.await(5,
TimeUnit.SECONDS));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ }
+ failedPrepares.incrementAndGet();
+ throw new IOException("intentional prepare
failure");
+ }
+ });
+ sinkWritersContext.put(failedIdentifier, new
TestSinkWriterContext());
+ }
+
+ MultiTableSinkWriter multiTableSinkWriter =
+ new MultiTableSinkWriter(
+ sinkWriters,
+ 2,
+ sinkWritersContext,
+ MultiTableFailurePolicy.CONTINUE_OTHER_TABLES,
+ JobMode.BATCH);
+ MultiTableCommitInfo commitInfo =
+ Assertions.assertTimeoutPreemptively(
+ Duration.ofSeconds(10),
+ () ->
multiTableSinkWriter.prepareCommit(1L).orElseThrow());
+ Assertions.assertEquals(2, failedPrepares.get());
+ Assertions.assertEquals(
+ Collections.singleton(healthyIdentifier),
commitInfo.getCommitInfo().keySet());
+
+ IOException closeException =
+ Assertions.assertThrows(IOException.class,
multiTableSinkWriter::close);
+
Assertions.assertTrue(closeException.getMessage().contains("test.failed0"));
+
Assertions.assertTrue(closeException.getMessage().contains("test.failed1"));
+
Assertions.assertTrue(closeException.getMessage().contains("phase=checkpoint"));
+ }
+
@Test
public void testPrepareCommitFatalRowErrorHandlingFailureFailsFast() {
Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new
HashMap<>();
@@ -570,28 +629,35 @@ public class MultiTableSinkWriterTest {
}
@Test
- public void testAggregatedFlushIsolatesFailedTableAndFlushesHealthyTable()
{
+ public void
testAggregatedFlushIsolatesConcurrentFailuresAndFlushesHealthyTable() {
Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new
HashMap<>();
Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new
HashMap<>();
Map<SinkIdentifier, SinkContextProxy> proxyContexts = new HashMap<>();
- SinkIdentifier failedIdentifier = SinkIdentifier.of("test.failed", 0);
- SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy",
1);
- sinkWriters.put(failedIdentifier, new TestSinkWriter());
+ SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy",
2);
sinkWriters.put(healthyIdentifier, new TestSinkWriter());
- sinkWritersContext.put(failedIdentifier, new TestSinkWriterContext());
sinkWritersContext.put(healthyIdentifier, new TestSinkWriterContext());
AtomicInteger failedFlushCount = new AtomicInteger();
- SinkContextProxy failedProxy = new SinkContextProxy(0, 2, new
TestSinkWriterContext());
- failedProxy.registerFlushAction(
- () -> {
- failedFlushCount.incrementAndGet();
- throw new IOException("intentional flush failure");
- });
+ CountDownLatch failedFlushesStarted = new CountDownLatch(2);
+ for (int queueIndex = 0; queueIndex < 2; queueIndex++) {
+ SinkIdentifier failedIdentifier =
+ SinkIdentifier.of("test.failed" + queueIndex, queueIndex);
+ sinkWriters.put(failedIdentifier, new TestSinkWriter());
+ sinkWritersContext.put(failedIdentifier, new
TestSinkWriterContext());
+ SinkContextProxy failedProxy =
+ new SinkContextProxy(queueIndex, 2, new
TestSinkWriterContext());
+ failedProxy.registerFlushAction(
+ () -> {
+ failedFlushCount.incrementAndGet();
+ failedFlushesStarted.countDown();
+ Assertions.assertTrue(failedFlushesStarted.await(5,
TimeUnit.SECONDS));
+ throw new IOException("intentional flush failure");
+ });
+ proxyContexts.put(failedIdentifier, failedProxy);
+ }
AtomicInteger healthyFlushCount = new AtomicInteger();
- SinkContextProxy healthyProxy = new SinkContextProxy(1, 2, new
TestSinkWriterContext());
+ SinkContextProxy healthyProxy = new SinkContextProxy(2, 2, new
TestSinkWriterContext());
healthyProxy.registerFlushAction(healthyFlushCount::incrementAndGet);
- proxyContexts.put(failedIdentifier, failedProxy);
proxyContexts.put(healthyIdentifier, healthyProxy);
MultiTableSinkWriter multiTableSinkWriter =
@@ -602,61 +668,178 @@ public class MultiTableSinkWriterTest {
MultiTableFailurePolicy.CONTINUE_OTHER_TABLES,
JobMode.BATCH);
- Assertions.assertDoesNotThrow(() ->
multiTableSinkWriter.aggregatedFlush(proxyContexts));
- Assertions.assertEquals(1, failedFlushCount.get());
+ Assertions.assertTimeoutPreemptively(
+ Duration.ofSeconds(10), () ->
multiTableSinkWriter.aggregatedFlush(proxyContexts));
+ Assertions.assertEquals(2, failedFlushCount.get());
Assertions.assertEquals(1, healthyFlushCount.get());
Assertions.assertDoesNotThrow(() ->
multiTableSinkWriter.aggregatedFlush(proxyContexts));
- Assertions.assertEquals(1, failedFlushCount.get());
+ Assertions.assertEquals(2, failedFlushCount.get());
Assertions.assertEquals(2, healthyFlushCount.get());
IOException closeException =
Assertions.assertThrows(IOException.class,
multiTableSinkWriter::close);
-
Assertions.assertTrue(closeException.getMessage().contains("test.failed"));
+
Assertions.assertTrue(closeException.getMessage().contains("test.failed0"));
+
Assertions.assertTrue(closeException.getMessage().contains("test.failed1"));
Assertions.assertTrue(closeException.getMessage().contains("phase=timer_flush"));
}
@Test
- public void testAggregatedFlushFailFastStopsAtFirstFailure() {
+ public void
testAggregatedFlushReportsFailureAndClosesQueuesIndependently() throws
Exception {
Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new
HashMap<>();
Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new
HashMap<>();
Map<SinkIdentifier, SinkContextProxy> proxyContexts = new HashMap<>();
SinkIdentifier failedIdentifier = SinkIdentifier.of("test.failed", 0);
- SinkIdentifier healthyIdentifier = SinkIdentifier.of("test.healthy",
1);
- sinkWriters.put(failedIdentifier, new TestSinkWriter());
- sinkWriters.put(healthyIdentifier, new TestSinkWriter());
+ SinkIdentifier slowIdentifier = SinkIdentifier.of("test.slow", 1);
+ SinkIdentifier siblingIdentifier = SinkIdentifier.of("test.sibling",
3);
+ CountDownLatch failedWriterClosed = new CountDownLatch(1);
+ AtomicBoolean slowFlushFinished = new AtomicBoolean();
+ AtomicBoolean slowWriterClosed = new AtomicBoolean();
+ sinkWriters.put(
+ failedIdentifier,
+ new TestSinkWriter() {
+ @Override
+ public void close() {
+ failedWriterClosed.countDown();
+ }
+ });
+ sinkWriters.put(
+ slowIdentifier,
+ new TestSinkWriter() {
+ @Override
+ public void close() {
+ Assertions.assertTrue(slowFlushFinished.get());
+ slowWriterClosed.set(true);
+ }
+ });
+ sinkWriters.put(siblingIdentifier, new TestSinkWriter());
+ sinkWritersContext.put(slowIdentifier, new TestSinkWriterContext());
+ sinkWritersContext.put(siblingIdentifier, new TestSinkWriterContext());
sinkWritersContext.put(failedIdentifier, new TestSinkWriterContext());
- sinkWritersContext.put(healthyIdentifier, new TestSinkWriterContext());
- AtomicInteger failedFlushCount = new AtomicInteger();
+ AtomicInteger slowQueueFlushCount = new AtomicInteger();
+ CountDownLatch slowFlushStarted = new CountDownLatch(1);
+ CompletableFuture<Void> releaseSlowFlush = new CompletableFuture<>();
+ IOException expectedFailure = new IOException("intentional flush
failure");
+ SinkContextProxy slowProxy = new SinkContextProxy(1, 2, new
TestSinkWriterContext());
+ slowProxy.registerFlushAction(
+ () -> {
+ // Block whichever child is visited first on this queue.
+ if (slowQueueFlushCount.incrementAndGet() == 1) {
+ slowFlushStarted.countDown();
+ releaseSlowFlush.join();
+ slowFlushFinished.set(true);
+ }
+ });
+ SinkContextProxy siblingProxy = new SinkContextProxy(3, 2, new
TestSinkWriterContext());
+ siblingProxy.registerFlushAction(slowProxy.getFlushAction());
SinkContextProxy failedProxy = new SinkContextProxy(0, 2, new
TestSinkWriterContext());
failedProxy.registerFlushAction(
() -> {
- failedFlushCount.incrementAndGet();
- throw new IOException("intentional flush failure");
+ Assertions.assertTrue(slowFlushStarted.await(5,
TimeUnit.SECONDS));
+ throw expectedFailure;
});
- AtomicInteger healthyFlushCount = new AtomicInteger();
- SinkContextProxy healthyProxy = new SinkContextProxy(1, 2, new
TestSinkWriterContext());
- healthyProxy.registerFlushAction(healthyFlushCount::incrementAndGet);
+ proxyContexts.put(slowIdentifier, slowProxy);
+ proxyContexts.put(siblingIdentifier, siblingProxy);
proxyContexts.put(failedIdentifier, failedProxy);
- proxyContexts.put(healthyIdentifier, healthyProxy);
MultiTableSinkWriter multiTableSinkWriter =
- new MultiTableSinkWriter(
- sinkWriters,
- 2,
- sinkWritersContext,
- MultiTableFailurePolicy.FAIL_FAST,
- JobMode.BATCH);
+ new MultiTableSinkWriter(sinkWriters, 2, sinkWritersContext);
+ FutureTask<Void> flush =
+ new FutureTask<>(
+ () -> {
+
multiTableSinkWriter.aggregatedFlush(proxyContexts);
+ return null;
+ });
+ Thread dispatcher = new Thread(flush);
+ dispatcher.setDaemon(true);
+ FutureTask<Void> close =
+ new FutureTask<>(
+ () -> {
+ multiTableSinkWriter.close();
+ return null;
+ });
+ Thread closer = new Thread(close);
+ closer.setDaemon(true);
+ try {
+ multiTableSinkWriter.write(buildRow("test.failed", 0));
+ dispatcher.start();
+ ExecutionException error =
+ Assertions.assertThrows(
+ ExecutionException.class, () -> flush.get(5,
TimeUnit.SECONDS));
+ Assertions.assertSame(expectedFailure, error.getCause());
- IOException flushException =
- Assertions.assertThrows(
- IOException.class,
- () ->
multiTableSinkWriter.aggregatedFlush(proxyContexts));
- Assertions.assertEquals("intentional flush failure",
flushException.getMessage());
- Assertions.assertEquals(1, failedFlushCount.get());
- Assertions.assertEquals(0, healthyFlushCount.get());
- Assertions.assertDoesNotThrow(multiTableSinkWriter::close);
+ closer.start();
+ Assertions.assertTrue(failedWriterClosed.await(5,
TimeUnit.SECONDS));
+ Assertions.assertFalse(slowWriterClosed.get());
+ } finally {
+ releaseSlowFlush.complete(null);
+ dispatcher.join(5000);
+ if (closer.getState() == Thread.State.NEW) {
+ closer.start();
+ }
+ close.get(5, TimeUnit.SECONDS);
+ }
+ Assertions.assertTrue(slowWriterClosed.get());
+ Assertions.assertEquals(1, slowQueueFlushCount.get());
+ }
+
+ @Test
+ public void testInterruptedAggregatedFlushDoesNotStartAnotherChildFlush()
throws Exception {
+ Map<SinkIdentifier, SinkWriter<SeaTunnelRow, ?, ?>> sinkWriters = new
HashMap<>();
+ Map<SinkIdentifier, SinkWriter.Context> sinkWritersContext = new
HashMap<>();
+ Map<SinkIdentifier, SinkContextProxy> proxyContexts = new HashMap<>();
+ AtomicInteger startedFlushes = new AtomicInteger();
+ CountDownLatch firstFlushStarted = new CountDownLatch(1);
+ CompletableFuture<Void> releaseFirstFlush = new CompletableFuture<>();
+ for (int index : new int[] {0, 2}) {
+ SinkIdentifier identifier = SinkIdentifier.of("test.table" +
index, index);
+ SinkWriter.Context context = new TestSinkWriterContext();
+ sinkWriters.put(identifier, new TestSinkWriter());
+ sinkWritersContext.put(identifier, context);
+ SinkContextProxy proxy = new SinkContextProxy(index, 2, context);
+ proxy.registerFlushAction(
+ () -> {
+ if (startedFlushes.incrementAndGet() == 1) {
+ firstFlushStarted.countDown();
+ releaseFirstFlush.join();
+ // Simulate a connector that consumes the
interrupt during I/O.
+ Thread.interrupted();
+ }
+ });
+ proxyContexts.put(identifier, proxy);
+ }
+
+ MultiTableSinkWriter multiTableSinkWriter =
+ new MultiTableSinkWriter(sinkWriters, 2, sinkWritersContext);
+ AtomicBoolean dispatcherInterrupted = new AtomicBoolean();
+ FutureTask<Void> flush =
+ new FutureTask<>(
+ () -> {
+ try {
+
multiTableSinkWriter.aggregatedFlush(proxyContexts);
+ return null;
+ } finally {
+
dispatcherInterrupted.set(Thread.currentThread().isInterrupted());
+ }
+ });
+ Thread dispatcher = new Thread(flush);
+ dispatcher.setDaemon(true);
+ dispatcher.start();
+ try {
+ Assertions.assertTrue(firstFlushStarted.await(5,
TimeUnit.SECONDS));
+ dispatcher.interrupt();
+ ExecutionException error =
+ Assertions.assertThrows(
+ ExecutionException.class, () -> flush.get(5,
TimeUnit.SECONDS));
+ Assertions.assertTrue(error.getCause() instanceof
InterruptedException);
+ Assertions.assertTrue(dispatcherInterrupted.get());
+ } finally {
+ releaseFirstFlush.complete(null);
+ dispatcher.join(5000);
+ multiTableSinkWriter.close();
+ }
+ Assertions.assertEquals(1, startedFlushes.get());
}
@Test