This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new bcac82ce5374 CAMEL-27088: camel-core - Delay, Throttle, Threads EIP:
fail the exchanges left in the thread pool when the route is removed (#30304)
bcac82ce5374 is described below
commit bcac82ce5374b5f458eb3de6f5347cd9053488c2
Author: allthingssecurity <[email protected]>
AuthorDate: Wed Nov 18 18:19:16 2026 +0530
CAMEL-27088: camel-core - Delay, Throttle, Threads EIP: fail the exchanges
left in the thread pool when the route is removed (#30304)
When a route is stopped after the shutdown timeout and then removed (or
reloaded), the Delay EIP, the
Throttle EIP with asyncDelayed and the Threads EIP shut down their own
thread pool with shutdownNow.
The exchanges whose task was still queued in that pool were dropped: their
callback was never called,
so the caller (such as a ProducerTemplate request) waited for ever, and the
exchange stayed in the
in-flight registry.
The Delay and Throttle EIPs now remember their queued tasks; the task and
the shutdown each claim a
task by removing it, and after shutting down its own thread pool the
shutdown fails the exchanges it
claimed with a RouteStoppingException. The Threads EIP rejects the tasks
that shutdownNow returns.
A thread pool of the user is not shut down by the EIP, so its tasks still
run, as before. The task of
the Throttle EIP stays a Callable, so the thread pool is called with the
same submit and schedule
methods as before.
When doShutdown claimed a call between its registration and a rejected
schedule or submit, the rejected path
completed the exchange a second time (and the Delay decremented its counter
twice): it now leaves the exchange alone
when it cannot claim the call. When the thread pool of the EIP was shut
down while the exchange was handed to it, the
CallerRuns policy dropped the task without an exception: the exchange is
now failed in the calling thread if it can
still be claimed. A comment in the Threads EIP notes that wrapped tasks of
instrumented thread pools are not failed.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../apache/camel/processor/AbstractThrottler.java | 90 +++++++++
.../processor/ConcurrentRequestsThrottler.java | 13 +-
.../camel/processor/DelayProcessorSupport.java | 47 +++++
.../apache/camel/processor/ThreadsProcessor.java | 14 +-
.../camel/processor/TotalRequestsThrottler.java | 13 +-
.../processor/AsyncDelayedRemoveRouteTest.java | 207 +++++++++++++++++++++
.../processor/AsyncDelayedShutdownRaceTest.java | 204 ++++++++++++++++++++
7 files changed, 583 insertions(+), 5 deletions(-)
diff --git
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/AbstractThrottler.java
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/AbstractThrottler.java
index 06a1b4bf2007..43c234eb5d24 100644
---
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/AbstractThrottler.java
+++
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/AbstractThrottler.java
@@ -16,6 +16,9 @@
*/
package org.apache.camel.processor;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
@@ -53,6 +56,10 @@ public abstract class AbstractThrottler extends
BaseProcessorSupport
protected boolean asyncDelayed;
protected boolean callerRunsWhenRejected = true;
protected volatile Expression maxRequestsExpression;
+ // the asynchronous tasks (asyncDelayed) that are queued in the thread
pool and not yet started: a task and
+ // doShutdown each claim a task by removing it, so an exchange is
processed, or failed when the thread pool is shut
+ // down, but never both
+ private final Set<AsyncCall> asyncCalls = ConcurrentHashMap.newKeySet();
AbstractThrottler(final ScheduledExecutorService asyncExecutor, final
boolean shutdownAsyncExecutor,
final CamelContext camelContext, final boolean
rejectExecution, Expression correlation,
@@ -82,6 +89,89 @@ public abstract class AbstractThrottler extends
BaseProcessorSupport
}
}
+ /**
+ * A task that processes the exchange again on the thread pool, unless
doShutdown failed the exchange before.
+ */
+ final class AsyncCall implements Callable<Boolean> {
+ private final Exchange exchange;
+ private final AsyncCallback callback;
+
+ private AsyncCall(Exchange exchange, AsyncCallback callback) {
+ this.exchange = exchange;
+ this.callback = callback;
+ }
+
+ // a Callable, as the task was before, so the thread pool is called
with the same submit and schedule methods
+ @Override
+ public Boolean call() {
+ if (asyncCalls.remove(this)) {
+ return process(exchange, callback);
+ }
+ return false;
+ }
+
+ /**
+ * The thread pool did not take the task (rejected), so it is no
longer queued.
+ *
+ * @return true if the caller still owns the exchange, false if
doShutdown has already failed the exchange and
+ * called the callback (the caller must then leave the
exchange alone)
+ */
+ boolean rejected() {
+ return asyncCalls.remove(this);
+ }
+
+ private void fail() {
+ // the exchange is no longer queued (a later throttler must
process it synchronously)
+ exchange.removeProperty(PROPERTY_EXCHANGE_STATE);
+ exchange.setException(
+ new RouteStoppingException("Throttled exchange is not
processed as the throttler is shut down"));
+ }
+ }
+
+ /**
+ * Creates the task to process the exchange again on the thread pool, and
remembers it until it starts.
+ */
+ AsyncCall newAsyncCall(Exchange exchange, AsyncCallback callback) {
+ AsyncCall call = new AsyncCall(exchange, callback);
+ asyncCalls.add(call);
+ return call;
+ }
+
+ /**
+ * Fails the exchanges whose asynchronous task has not started yet. They
would never be processed once the thread
+ * pool is shut down (such as when the route is removed after the shutdown
timeout ran out), and their callers would
+ * wait for ever. Only called once doShutdown shut down the thread pool of
this EIP: a thread pool of the user still
+ * runs the tasks.
+ */
+ void failAsyncCalls() {
+ for (AsyncCall call : asyncCalls) {
+ if (asyncCalls.remove(call)) {
+ LOG.debug("Throttled exchange is not processed as the
throttler is shut down, exchangeId: {}",
+ call.exchange.getExchangeId());
+ call.fail();
+ call.callback.done(false);
+ }
+ }
+ }
+
+ /**
+ * Fails the exchange in the calling thread when the thread pool of this
EIP was shut down while the task was handed
+ * to it. With the CallerRuns rejected policy (the default) a thread pool
that is shut down drops the task without
+ * an exception, and doShutdown may have failed the queued tasks before
this one was added.
+ *
+ * @return true if the exchange was failed here and the callback was
called synchronously
+ */
+ boolean failIfShutdown(AsyncCall call) {
+ if (shutdownAsyncExecutor && asyncExecutor.isShutdown() &&
asyncCalls.remove(call)) {
+ LOG.debug("Throttled exchange is not processed as the throttler is
shut down, exchangeId: {}",
+ call.exchange.getExchangeId());
+ call.fail();
+ call.callback.done(true);
+ return true;
+ }
+ return false;
+ }
+
protected static boolean handleInterrupt(
Exchange exchange, AsyncCallback callback, InterruptedException e,
boolean doneSync) {
// determine if we can still run, or the camel context is forcing a
shutdown
diff --git
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/ConcurrentRequestsThrottler.java
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/ConcurrentRequestsThrottler.java
index f1d55bfc0ef2..e5eff47cf5e2 100644
---
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/ConcurrentRequestsThrottler.java
+++
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/ConcurrentRequestsThrottler.java
@@ -183,14 +183,20 @@ public class ConcurrentRequestsThrottler extends
AbstractThrottler {
*/
protected boolean processAsynchronously(
final Exchange exchange, final AsyncCallback callback,
ThrottlingState throttlingState) {
+ AsyncCall call = newAsyncCall(exchange, callback);
try {
if (LOG.isTraceEnabled()) {
exchange.setProperty(PROPERTY_EXCHANGE_QUEUED_TIME,
TimeSource.nanoTime());
}
exchange.setProperty(PROPERTY_EXCHANGE_STATE, State.ASYNC);
- asyncExecutor.submit(() -> process(exchange, callback));
- return false;
+ asyncExecutor.submit(call);
+ // a thread pool that is shut down drops the task without an
exception (CallerRuns)
+ return failIfShutdown(call);
} catch (final RejectedExecutionException e) {
+ if (!call.rejected()) {
+ // doShutdown has already failed the exchange and called the
callback
+ return false;
+ }
if (isCallerRunsWhenRejected()) {
if (LOG.isDebugEnabled()) {
LOG.debug("AsyncExecutor is full, rejected exchange will
run in the current thread, exchangeId: {}",
@@ -214,6 +220,9 @@ public class ConcurrentRequestsThrottler extends
AbstractThrottler {
protected void doShutdown() throws Exception {
if (shutdownAsyncExecutor && asyncExecutor != null) {
camelContext.getExecutorServiceManager().shutdownNow(asyncExecutor);
+ // the tasks still queued in the thread pool that was shut down
never run (a thread pool of the user
+ // is not shut down here, and still runs them)
+ failAsyncCalls();
}
states.clear();
super.doShutdown();
diff --git
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/DelayProcessorSupport.java
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/DelayProcessorSupport.java
index d5b2c9731658..c6f0838fc38f 100644
---
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/DelayProcessorSupport.java
+++
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/DelayProcessorSupport.java
@@ -16,6 +16,8 @@
*/
package org.apache.camel.processor;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -46,6 +48,9 @@ public abstract class DelayProcessorSupport extends
BaseDelegateProcessorSupport
private boolean asyncDelayed = true;
private boolean callerRunsWhenRejected = true;
private final AtomicInteger delayedCount = new AtomicInteger();
+ // the delayed tasks that are scheduled and not yet started: a task and
doShutdown each claim a task by removing
+ // it, so an exchange is routed on, or failed when the thread pool is shut
down, but never both
+ private final Set<ProcessCall> scheduledCalls =
ConcurrentHashMap.newKeySet();
private final class ProcessCall implements Runnable {
private final Exchange exchange;
@@ -58,6 +63,10 @@ public abstract class DelayProcessorSupport extends
BaseDelegateProcessorSupport
@Override
public void run() {
+ if (!scheduledCalls.remove(this)) {
+ // the exchange was already failed by doShutdown
+ return;
+ }
// we are running now so decrement the counter
delayedCount.decrementAndGet();
@@ -120,15 +129,29 @@ public abstract class DelayProcessorSupport extends
BaseDelegateProcessorSupport
// and increment the counter (we decrement the counter when we run
the ProcessCall)
delayedCount.incrementAndGet();
ProcessCall call = new ProcessCall(exchange, callback);
+ scheduledCalls.add(call);
try {
if (LOG.isTraceEnabled()) {
LOG.trace("Scheduling delayed task to run in {} millis for
exchangeId: {}",
delay, exchange.getExchangeId());
}
executorService.schedule(call, delay, TimeUnit.MILLISECONDS);
+ if (shutdownExecutorService && executorService.isShutdown() &&
scheduledCalls.remove(call)) {
+ // the thread pool was shut down while the task was handed
to it: with the CallerRuns rejected
+ // policy (the default) it drops the task without an
exception, and doShutdown may have failed
+ // the scheduled tasks before this one was added
+ delayedCount.decrementAndGet();
+ failNotRun(exchange);
+ callback.done(true);
+ return true;
+ }
// tell Camel routing engine we continue routing asynchronous
return false;
} catch (RejectedExecutionException e) {
+ if (!scheduledCalls.remove(call)) {
+ // doShutdown has already failed the exchange and called
the callback
+ return false;
+ }
// we were not allowed to run the ProcessCall, so need to
decrement the counter here
delayedCount.decrementAndGet();
if (isCallerRunsWhenRejected()) {
@@ -273,10 +296,34 @@ public abstract class DelayProcessorSupport extends
BaseDelegateProcessorSupport
super.doStart();
}
+ /**
+ * Fails the exchanges whose delayed task has not started yet. They would
never be routed on once the thread pool is
+ * shut down (such as when the route is removed after the shutdown timeout
ran out), and their callers would wait
+ * for ever. Only called once doShutdown shut down the thread pool of this
EIP: a thread pool of the user still runs
+ * the tasks.
+ */
+ private void failScheduledCalls() {
+ for (ProcessCall call : scheduledCalls) {
+ if (scheduledCalls.remove(call)) {
+ delayedCount.decrementAndGet();
+ failNotRun(call.exchange);
+ call.callback.done(false);
+ }
+ }
+ }
+
+ private static void failNotRun(Exchange exchange) {
+ LOG.debug("Delayed task is not run as the processor is shut down, for
exchangeId: {}", exchange.getExchangeId());
+ exchange.setException(new RouteStoppingException("The delayed exchange
is not run: the Delay is shut down"));
+ }
+
@Override
protected void doShutdown() throws Exception {
if (shutdownExecutorService && executorService != null) {
camelContext.getExecutorServiceManager().shutdownNow(executorService);
+ // the tasks still queued in the thread pool that was shut down
never run (a thread pool of the user
+ // is not shut down here, and still runs them)
+ failScheduledCalls();
}
super.doShutdown();
}
diff --git
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/ThreadsProcessor.java
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/ThreadsProcessor.java
index c2cbe8ae1df2..c230e7cba539 100644
---
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/ThreadsProcessor.java
+++
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/ThreadsProcessor.java
@@ -16,6 +16,7 @@
*/
package org.apache.camel.processor;
+import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
@@ -208,7 +209,18 @@ public class ThreadsProcessor extends BaseProcessorSupport
implements IdAware, R
@Override
protected void doShutdown() throws Exception {
if (shutdownExecutorService) {
-
camelContext.getExecutorServiceManager().shutdownNow(executorService);
+ List<Runnable> dropped =
camelContext.getExecutorServiceManager().shutdownNow(executorService);
+ if (dropped != null) {
+ // the exchanges still queued in the thread pool are never
processed: reject them so their callers
+ // do not wait for ever. A thread pool factory that wraps the
tasks (such as the instrumented thread
+ // pools of camel-opentelemetry or camel-micrometer) hands
back its own wrappers here, which are not
+ // Rejectable, so those exchanges are not failed here
+ for (Runnable task : dropped) {
+ if (task instanceof Rejectable rejectable) {
+ rejectable.reject();
+ }
+ }
+ }
}
super.doShutdown();
}
diff --git
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/TotalRequestsThrottler.java
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/TotalRequestsThrottler.java
index 8a28c1809483..688cbfe901ec 100644
---
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/TotalRequestsThrottler.java
+++
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/TotalRequestsThrottler.java
@@ -191,6 +191,7 @@ public class TotalRequestsThrottler extends
AbstractThrottler {
*/
protected boolean processAsynchronously(
final Exchange exchange, final AsyncCallback callback,
ThrottlingState throttlingState) {
+ AsyncCall call = newAsyncCall(exchange, callback);
try {
if (LOG.isTraceEnabled()) {
exchange.setProperty(PROPERTY_EXCHANGE_QUEUED_TIMESTAMP,
TimeSource.currentTimeMillis());
@@ -201,9 +202,14 @@ public class TotalRequestsThrottler extends
AbstractThrottler {
// exchanges and not yet returned, so try again after one period
long delay = next != null
? next.getDelay(TimeUnit.NANOSECONDS) :
TimeUnit.MILLISECONDS.toNanos(getTimePeriodMillis());
- asyncExecutor.schedule(() -> process(exchange, callback), delay,
TimeUnit.NANOSECONDS);
- return false;
+ asyncExecutor.schedule(call, delay, TimeUnit.NANOSECONDS);
+ // a thread pool that is shut down drops the task without an
exception (CallerRuns)
+ return failIfShutdown(call);
} catch (final RejectedExecutionException e) {
+ if (!call.rejected()) {
+ // doShutdown has already failed the exchange and called the
callback
+ return false;
+ }
if (isCallerRunsWhenRejected()) {
if (LOG.isDebugEnabled()) {
LOG.debug("AsyncExecutor is full, rejected exchange will
run in the current thread, exchangeId: {}",
@@ -227,6 +233,9 @@ public class TotalRequestsThrottler extends
AbstractThrottler {
protected void doShutdown() throws Exception {
if (shutdownAsyncExecutor && asyncExecutor != null) {
camelContext.getExecutorServiceManager().shutdownNow(asyncExecutor);
+ // the tasks still queued in the thread pool that was shut down
never run (a thread pool of the user
+ // is not shut down here, and still runs them)
+ failAsyncCalls();
}
states.clear();
super.doShutdown();
diff --git
a/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedRemoveRouteTest.java
b/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedRemoveRouteTest.java
new file mode 100644
index 000000000000..c44f7636d9d6
--- /dev/null
+++
b/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedRemoveRouteTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.Exchange;
+import org.apache.camel.RouteStoppingException;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.builder.ThreadPoolProfileBuilder;
+import org.apache.camel.spi.ThreadPoolProfile;
+import org.junit.jupiter.api.Test;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * An exchange that waits in the thread pool of a Delay, Throttle
(asyncDelayed) or Threads EIP when its route is
+ * stopped after the shutdown timeout and then removed must be completed
(failed), not dropped with the thread pool. A
+ * thread pool of the user is not shut down with the route, so its tasks still
run and complete the exchange.
+ */
+class AsyncDelayedRemoveRouteTest extends ContextTestSupport {
+
+ private final CountDownLatch latch = new CountDownLatch(1);
+
+ @Override
+ public boolean isUseRouteBuilder() {
+ return false;
+ }
+
+ @Test
+ void testDelayRemoveRoute() throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+
from("direct:delay").routeId("delay").delay(60000).asyncDelayed().to("mock:result");
+ }
+ });
+ context.start();
+ assertExchangeCompletedOnRemove("direct:delay", "delay");
+ }
+
+ @Test
+ void testThrottleRemoveRoute() throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:throttle").routeId("throttle")
+
.throttle(1).totalRequestsMode().timePeriodMillis(60000).asyncDelayed()
+ .to("mock:result");
+ }
+ });
+ context.start();
+ // the first exchange takes the only permit of the period
+ template.sendBody("direct:throttle", "first");
+ assertExchangeCompletedOnRemove("direct:throttle", "throttle");
+ }
+
+ @Test
+ void testConcurrentThrottleRemoveRoute() throws Exception {
+ // one thread in the thread pool of the throttler
+ ThreadPoolProfile one = new
ThreadPoolProfileBuilder("one").poolSize(1).maxPoolSize(1).build();
+ context.getExecutorServiceManager().registerThreadPoolProfile(one);
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:concurrent").routeId("concurrent")
+
.throttle(1).concurrentRequestsMode().asyncDelayed().executorService("one")
+ .process(e -> {
+ if ("first".equals(e.getMessage().getBody())) {
+ latch.await(20, TimeUnit.SECONDS);
+ }
+ });
+ }
+ });
+ context.start();
+ // the first exchange keeps the only permit, the second one waits for
it on the only thread of the pool
+ CompletableFuture<Exchange> first =
template.asyncSend("direct:concurrent", e -> e.getMessage().setBody("first"));
+ await().atMost(5, TimeUnit.SECONDS).until(() ->
context.getInflightRepository().size("concurrent") == 1);
+ CompletableFuture<Exchange> second =
template.asyncSend("direct:concurrent", e -> e.getMessage().setBody("second"));
+ await().atMost(5, TimeUnit.SECONDS).until(() ->
context.getInflightRepository().size("concurrent") == 2);
+ try {
+ // the third one is queued in the thread pool
+ assertExchangeCompletedOnRemove("direct:concurrent", "concurrent",
3);
+ assertNotNull(second.get(10, TimeUnit.SECONDS).getException());
+ } finally {
+ latch.countDown();
+ }
+ // the first one completes too, cut off by the forced stop
+ assertInstanceOf(RouteStoppingException.class, first.get(10,
TimeUnit.SECONDS).getException());
+ assertEquals(0, context.getInflightRepository().size());
+ }
+
+ @Test
+ void testThreadsRemoveRoute() throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:threads").routeId("threads")
+ .threads(1, 1).maxQueueSize(10)
+ .process(e -> {
+ if ("first".equals(e.getMessage().getBody())) {
+ latch.await(20, TimeUnit.SECONDS);
+ }
+ });
+ }
+ });
+ context.start();
+ // the first exchange keeps the only thread of the pool busy
+ CompletableFuture<Exchange> first =
template.asyncSend("direct:threads", e -> e.getMessage().setBody("first"));
+ await().atMost(5, TimeUnit.SECONDS).until(() ->
context.getInflightRepository().size("threads") == 1);
+ try {
+ // the second one is queued in the thread pool
+ assertExchangeCompletedOnRemove("direct:threads", "threads", 2);
+ } finally {
+ latch.countDown();
+ }
+ // the first one was interrupted by the shutdown of the thread pool
+ assertNotNull(first.get(10, TimeUnit.SECONDS).getException());
+ assertEquals(0, context.getInflightRepository().size());
+ }
+
+ @Test
+ void testDelayWithThreadPoolOfTheUserRemoveRoute() throws Exception {
+ ScheduledExecutorService pool =
Executors.newSingleThreadScheduledExecutor();
+ try {
+ context.getRegistry().bind("userPool", pool);
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+
from("direct:user").routeId("user").delay(10).asyncDelayed().executorService("userPool")
+ .to("mock:result");
+ }
+ });
+ context.start();
+ // the only thread of the pool is busy, so the delayed task waits
in its queue
+ CountDownLatch busy = new CountDownLatch(1);
+ pool.execute(() -> {
+ busy.countDown();
+ try {
+ latch.await(20, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+ assertTrue(busy.await(5, TimeUnit.SECONDS));
+ CompletableFuture<Exchange> future =
template.asyncSend("direct:user", e -> e.getMessage().setBody("waiting"));
+ await().atMost(5, TimeUnit.SECONDS).until(() ->
context.getInflightRepository().size("user") == 1);
+
+ context.getRouteController().stopRoute("user", 100,
TimeUnit.MILLISECONDS);
+ context.removeRoute("user");
+
+ // the route does not shut down the thread pool of the user, so
the task is still queued there and is not
+ // failed by the removal
+ await().during(300, TimeUnit.MILLISECONDS).atMost(2,
TimeUnit.SECONDS).until(() -> !future.isDone());
+ assertFalse(pool.isShutdown());
+
+ // the task runs once the thread is free, and fails the exchange
as its route was removed
+ latch.countDown();
+ assertInstanceOf(RouteStoppingException.class, future.get(10,
TimeUnit.SECONDS).getException());
+ assertEquals(0, context.getInflightRepository().size());
+ } finally {
+ latch.countDown();
+ pool.shutdownNow();
+ }
+ }
+
+ private void assertExchangeCompletedOnRemove(String uri, String routeId)
throws Exception {
+ assertExchangeCompletedOnRemove(uri, routeId, 1);
+ assertEquals(0, context.getInflightRepository().size());
+ }
+
+ private void assertExchangeCompletedOnRemove(String uri, String routeId,
int inflight) throws Exception {
+ CompletableFuture<Exchange> future = template.asyncSend(uri, e ->
e.getMessage().setBody("waiting"));
+ await().atMost(5, TimeUnit.SECONDS).until(() ->
context.getInflightRepository().size(routeId) == inflight);
+
+ // the shutdown timeout runs out, the route is stopped by force, then
removed
+ context.getRouteController().stopRoute(routeId, 100,
TimeUnit.MILLISECONDS);
+ context.removeRoute(routeId);
+
+ Exchange out = future.get(10, TimeUnit.SECONDS);
+ assertInstanceOf(RouteStoppingException.class, out.getException());
+ }
+}
diff --git
a/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedShutdownRaceTest.java
b/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedShutdownRaceTest.java
new file mode 100644
index 000000000000..d476b7531c51
--- /dev/null
+++
b/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedShutdownRaceTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.Exchange;
+import org.apache.camel.RouteStoppingException;
+import org.apache.camel.Service;
+import org.apache.camel.support.DefaultExchange;
+import org.apache.camel.support.builder.ExpressionBuilder;
+import org.apache.camel.support.service.ServiceHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The thread pool of a Delay or Throttle (asyncDelayed) EIP is shut down by
the EIP while an exchange is being handed
+ * to it. The exchange must be completed exactly once: either by doShutdown,
or by the thread that hands it over, never
+ * by both, and never by none.
+ */
+class AsyncDelayedShutdownRaceTest extends ContextTestSupport {
+
+ private final AtomicInteger done = new AtomicInteger();
+ private Exchange exchange;
+ private HookPool pool;
+
+ @BeforeEach
+ void createExchange() {
+ exchange = new DefaultExchange(context);
+ }
+
+ @AfterEach
+ void shutdownPool() {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ }
+
+ @Override
+ public boolean isUseRouteBuilder() {
+ return false;
+ }
+
+ @Test
+ void testDelayRejectedAfterShutdownFailedTheExchange() throws Exception {
+ // the EIP is shut down after the task was registered and before the
pool takes it, so doShutdown fails the
+ // exchange, then the pool that is shut down rejects the task
+ pool = new HookPool(new ThreadPoolExecutor.AbortPolicy());
+ Delayer delayer = new Delayer(context, e -> {
+ }, ExpressionBuilder.constantExpression(60000L), pool, true);
+ ServiceHelper.startService(delayer);
+ pool.beforeSchedule = () ->
ServiceHelper.stopAndShutdownService(delayer);
+
+ boolean sync = delayer.process(exchange, doneSync ->
done.incrementAndGet());
+
+ assertFalse(sync);
+ assertCompletedOnce();
+ assertEquals(0, delayer.getDelayedCount());
+ }
+
+ @Test
+ void testDelayDroppedByShutDownPool() throws Exception {
+ // doShutdown has failed the scheduled tasks before this one was
registered, and the pool that is shut down
+ // drops it without an exception (CallerRuns)
+ pool = new HookPool(new ThreadPoolExecutor.CallerRunsPolicy());
+ Delayer delayer = new Delayer(context, e -> {
+ }, ExpressionBuilder.constantExpression(60000L), pool, true);
+ ServiceHelper.startService(delayer);
+ ServiceHelper.stopAndShutdownService(delayer);
+
+ boolean sync = delayer.processDelay(exchange, doneSync ->
done.incrementAndGet(), 60000L);
+
+ assertTrue(sync);
+ assertCompletedOnce();
+ assertEquals(0, delayer.getDelayedCount());
+ }
+
+ @Test
+ void testConcurrentThrottlerRejectedAfterShutdownFailedTheExchange()
throws Exception {
+ pool = new HookPool(new ThreadPoolExecutor.AbortPolicy());
+ ConcurrentRequestsThrottler throttler = new
ConcurrentRequestsThrottler(
+ context, ExpressionBuilder.constantExpression(1), pool, true,
false, null);
+ throttler.setAsyncDelayed(true);
+ assertRejectedAfterShutdownCompletesOnce(throttler,
+ () -> throttler.processAsynchronously(exchange, doneSync ->
done.incrementAndGet(), null));
+ }
+
+ @Test
+ void testTotalThrottlerRejectedAfterShutdownFailedTheExchange() throws
Exception {
+ pool = new HookPool(new ThreadPoolExecutor.AbortPolicy());
+ TotalRequestsThrottler throttler = new TotalRequestsThrottler(
+ context, ExpressionBuilder.constantExpression(1), 60000L,
pool, true, false, null);
+ throttler.setAsyncDelayed(true);
+ TotalRequestsThrottler.ThrottlingState state = throttler.new
ThrottlingState("test");
+ assertRejectedAfterShutdownCompletesOnce(throttler,
+ () -> throttler.processAsynchronously(exchange, doneSync ->
done.incrementAndGet(), state));
+ }
+
+ @Test
+ void testConcurrentThrottlerDroppedByShutDownPool() throws Exception {
+ pool = new HookPool(new ThreadPoolExecutor.CallerRunsPolicy());
+ ConcurrentRequestsThrottler throttler = new
ConcurrentRequestsThrottler(
+ context, ExpressionBuilder.constantExpression(1), pool, true,
false, null);
+ throttler.setAsyncDelayed(true);
+ assertDroppedByShutDownPoolCompletesOnce(throttler,
+ () -> throttler.processAsynchronously(exchange, doneSync ->
done.incrementAndGet(), null));
+ }
+
+ @Test
+ void testTotalThrottlerDroppedByShutDownPool() throws Exception {
+ pool = new HookPool(new ThreadPoolExecutor.CallerRunsPolicy());
+ TotalRequestsThrottler throttler = new TotalRequestsThrottler(
+ context, ExpressionBuilder.constantExpression(1), 60000L,
pool, true, false, null);
+ throttler.setAsyncDelayed(true);
+ TotalRequestsThrottler.ThrottlingState state = throttler.new
ThrottlingState("test");
+ assertDroppedByShutDownPoolCompletesOnce(throttler,
+ () -> throttler.processAsynchronously(exchange, doneSync ->
done.incrementAndGet(), state));
+ }
+
+ private void assertRejectedAfterShutdownCompletesOnce(Service throttler,
Callable<Boolean> handOver)
+ throws Exception {
+ ServiceHelper.startService(throttler);
+ pool.beforeSchedule = () ->
ServiceHelper.stopAndShutdownService(throttler);
+
+ boolean sync = handOver.call();
+
+ assertFalse(sync);
+ assertCompletedOnce();
+ }
+
+ private void assertDroppedByShutDownPoolCompletesOnce(Service throttler,
Callable<Boolean> handOver)
+ throws Exception {
+ ServiceHelper.startService(throttler);
+ ServiceHelper.stopAndShutdownService(throttler);
+
+ boolean sync = handOver.call();
+
+ assertTrue(sync);
+ assertCompletedOnce();
+ }
+
+ private void assertCompletedOnce() {
+ assertEquals(1, done.get(), "the callback must be called exactly
once");
+ assertInstanceOf(RouteStoppingException.class,
exchange.getException());
+ }
+
+ /**
+ * A scheduled thread pool that runs a hook just before it takes a task.
+ */
+ private static final class HookPool extends ScheduledThreadPoolExecutor {
+ private volatile Runnable beforeSchedule;
+
+ HookPool(RejectedExecutionHandler handler) {
+ super(1, handler);
+ }
+
+ @Override
+ public ScheduledFuture<?> schedule(Runnable command, long delay,
TimeUnit unit) {
+ runHook();
+ return super.schedule(command, delay, unit);
+ }
+
+ @Override
+ public <V> ScheduledFuture<V> schedule(Callable<V> callable, long
delay, TimeUnit unit) {
+ runHook();
+ return super.schedule(callable, delay, unit);
+ }
+
+ private void runHook() {
+ Runnable hook = beforeSchedule;
+ beforeSchedule = null;
+ if (hook != null) {
+ hook.run();
+ }
+ }
+ }
+}