This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 72fe7104d4f8 CAMEL-27086: camel-zookeeper - a consumer that is stopped
and started again consumes again (#30310)
72fe7104d4f8 is described below
commit 72fe7104d4f8fb31afc1f44545b9d5abd02404ea
Author: allthingssecurity <[email protected]>
AuthorDate: Wed Nov 18 15:04:12 2026 +0530
CAMEL-27086: camel-zookeeper - a consumer that is stopped and started again
consumes again (#30310)
ZooKeeperConsumer set shuttingDown in doStop and never reset it. A consumer
that was stopped and started again in place (a route policy that stops and
starts the consumer, such as ThrottlingInflightRoutePolicy or the circuit
breaker of ThrottlingExceptionRoutePolicy, or JMX) skipped the operations of
its sequence at the second start: its new operations thread waited for ever
on
an empty queue and the route received nothing more, without an error. The
thread of the first start was not interrupted by the stop; it stayed blocked
on the queue or on the watch of the closed connection, and could take the
operations of a later start.
The start now clears shuttingDown. The stop ends the operations thread of
that
start: it is marked stopped and interrupted (unless the stop runs on that
thread, as a route policy may do when an exchange is done), it does not add
a
copy of its operation or start the sequence again, and an interrupt caused
by
the stop is not reported as an error.
A graceful shutdown stops the consumer (it is not Suspendable) and then
waits for the exchanges in flight. The stop
now interrupts the operations thread only while it waits for an operation
or a watch, not while it routes an
exchange: after the exchange the loop sees the stop and ends. The test
helpers only count the threads of their own
CamelContext.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../component/zookeeper/ZooKeeperConsumer.java | 107 ++++++++++-
.../zookeeper/ZooKeeperConsumerRestartTest.java | 208 +++++++++++++++++++++
2 files changed, 306 insertions(+), 9 deletions(-)
diff --git
a/components/camel-zookeeper/src/main/java/org/apache/camel/component/zookeeper/ZooKeeperConsumer.java
b/components/camel-zookeeper/src/main/java/org/apache/camel/component/zookeeper/ZooKeeperConsumer.java
index 275c661ecf2e..b497eb20d3d8 100644
---
a/components/camel-zookeeper/src/main/java/org/apache/camel/component/zookeeper/ZooKeeperConsumer.java
+++
b/components/camel-zookeeper/src/main/java/org/apache/camel/component/zookeeper/ZooKeeperConsumer.java
@@ -20,6 +20,8 @@ import java.time.Duration;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
@@ -56,6 +58,7 @@ public class ZooKeeperConsumer extends DefaultConsumer {
private ExecutorService executor;
private ScheduledExecutorService taskExecutor;
private volatile boolean shuttingDown;
+ private volatile OperationsExecutor opsService;
public ZooKeeperConsumer(ZooKeeperEndpoint endpoint, Processor processor) {
super(endpoint, processor);
@@ -66,6 +69,8 @@ public class ZooKeeperConsumer extends DefaultConsumer {
@Override
protected void doStart() throws Exception {
super.doStart();
+ // the consumer can be stopped and started again (by a route policy or
JMX)
+ shuttingDown = false;
connection = zkm.getConnection();
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Connected to Zookeeper cluster %s",
configuration.getConnectString()));
@@ -77,7 +82,7 @@ public class ZooKeeperConsumer extends DefaultConsumer {
taskExecutor =
getEndpoint().getCamelContext().getExecutorServiceManager()
.newSingleThreadScheduledExecutor(this,
"ZooKeeperReconnectTask");
- OperationsExecutor opsService = new OperationsExecutor();
+ opsService = new OperationsExecutor();
executor.submit(opsService);
}
@@ -92,6 +97,13 @@ public class ZooKeeperConsumer extends DefaultConsumer {
getEndpoint().getCamelContext().getExecutorServiceManager().shutdown(taskExecutor);
taskExecutor = null;
}
+ OperationsExecutor ops = opsService;
+ if (ops != null) {
+ // the thread of this start must end: it may wait for an operation
or a watch that never comes, and it
+ // must not take the operations of a later start
+ ops.stop();
+ opsService = null;
+ }
getEndpoint().getCamelContext().getExecutorServiceManager().shutdown(executor);
zkm.shutdown();
}
@@ -141,9 +153,71 @@ public class ZooKeeperConsumer extends DefaultConsumer {
private WatchedEvent watchedEvent;
+ private volatile boolean stopped;
+
+ private volatile Thread thread;
+
+ // guards processing, and the check of stopped before an exchange is
routed
+ private final Lock lock = new ReentrantLock();
+
+ // guarded by lock: an exchange is being routed on this thread
+ private boolean processing;
+
+ /**
+ * Ends this thread: the consumer stopped. The thread is interrupted
while it waits for an operation or a watch,
+ * but not while it routes an exchange (a graceful shutdown stops the
consumer first, then waits for the
+ * exchange to complete), and not when the consumer is stopped from
this thread (such as by a route policy when
+ * an exchange is done). After the exchange, the loop sees the stop
and ends.
+ */
+ void stop() {
+ lock.lock();
+ try {
+ stopped = true;
+ Thread t = thread;
+ if (!processing && t != null && t != Thread.currentThread()) {
+ t.interrupt();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private boolean isRunning() {
+ return !stopped && isRunAllowed();
+ }
+
+ /**
+ * @return false if the consumer was stopped, and no exchange must be
routed
+ */
+ private boolean beginProcessing() {
+ lock.lock();
+ try {
+ if (stopped) {
+ return false;
+ }
+ processing = true;
+ return true;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private void endProcessing() {
+ lock.lock();
+ try {
+ processing = false;
+ } finally {
+ lock.unlock();
+ }
+ }
+
@Override
public void run() {
- while (isRunAllowed()) {
+ thread = Thread.currentThread();
+ if (stopped) {
+ return;
+ }
+ while (isRunning()) {
try {
current = operations.take();
if (LOG.isTraceEnabled()) {
@@ -161,11 +235,23 @@ public class ZooKeeperConsumer extends DefaultConsumer {
watchedEvent = ZooKeeperUtils.getWatchedEvent(current);
}
if (result != null && current.shouldProduceExchange()) {
- getProcessor().process(createExchange(node, result,
watchedEvent));
+ if (!beginProcessing()) {
+ // stopped while the operation completed: the
exchange is not created
+ break;
+ }
+ try {
+ getProcessor().process(createExchange(node,
result, watchedEvent));
+ } finally {
+ endProcessing();
+ }
watchedEvent = null;
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
+ if (!isRunning()) {
+ // interrupted by the stop of the consumer
+ break;
+ }
handleException(e);
restarted = backoffAndThenRestart();
} catch (Exception e) {
@@ -174,7 +260,7 @@ public class ZooKeeperConsumer extends DefaultConsumer {
} finally {
// after a restart the queue holds the operations of the
new sequence; a copy of the failed one
// (which may use a handle whose session expired) must not
be added to them
- if (configuration.isRepeat() && !restarted) {
+ if (configuration.isRepeat() && !restarted && isRunning())
{
try {
operations.offer(current.createCopy());
} catch (Exception e) {
@@ -193,7 +279,7 @@ public class ZooKeeperConsumer extends DefaultConsumer {
*/
private boolean backoffAndThenRestart() {
try {
- if (isRunAllowed()) {
+ if (isRunning()) {
// Use Camel's task API for reconnection backoff delay
instead of Thread.sleep()
// We use initialDelay for the actual delay, and
maxIterations(1) to run once
Tasks.backgroundTask()
@@ -208,12 +294,15 @@ public class ZooKeeperConsumer extends DefaultConsumer {
.build()
.run(getEndpoint().getCamelContext(), () -> true);
- if (isRunAllowed()) {
+ if (isRunning()) {
connection = zkm.getConnection();
- if (!isRunAllowed()) {
+ if (!isRunning()) {
// stopped while the connection was created:
doStop may already have closed the holder,
- // so close the new handle (and its client
threads) here
- zkm.shutdown();
+ // so close the new handle (and its client
threads) here, unless the consumer was started
+ // again and uses it
+ if (!isRunAllowed()) {
+ zkm.shutdown();
+ }
return false;
}
initializeConsumer();
diff --git
a/components/camel-zookeeper/src/test/java/org/apache/camel/component/zookeeper/ZooKeeperConsumerRestartTest.java
b/components/camel-zookeeper/src/test/java/org/apache/camel/component/zookeeper/ZooKeeperConsumerRestartTest.java
new file mode 100644
index 000000000000..8cf5c5a0c0b2
--- /dev/null
+++
b/components/camel-zookeeper/src/test/java/org/apache/camel/component/zookeeper/ZooKeeperConsumerRestartTest.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.zookeeper;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.camel.Consumer;
+import org.apache.camel.RuntimeCamelException;
+import org.apache.camel.StatefulService;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import
org.apache.camel.component.zookeeper.operations.FutureEventDrivenOperation;
+import org.apache.camel.support.service.ServiceHelper;
+import org.apache.camel.test.AvailablePortFinder;
+import org.apache.camel.test.junit6.CamelTestSupport;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.server.NIOServerCnxnFactory;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The consumer is stopped and started again in place, as a route policy
(ThrottlingInflightRoutePolicy,
+ * ThrottlingExceptionRoutePolicy) or JMX does: it must consume again, on one
operations thread.
+ */
+class ZooKeeperConsumerRestartTest extends CamelTestSupport {
+
+ private static final String NODE = "/camel-consumer-restart";
+
+ @TempDir
+ Path dataDir;
+
+ private final int port = AvailablePortFinder.getNextAvailable();
+
+ private ZooKeeperServer server;
+ private NIOServerCnxnFactory factory;
+ private ZooKeeper client;
+
+ @Override
+ protected void doPreSetup() throws Exception {
+ File dir = dataDir.toFile();
+ server = new ZooKeeperServer(dir, dir, 2000);
+ factory = new NIOServerCnxnFactory();
+ factory.configure(new InetSocketAddress("127.0.0.1", port), 60);
+ factory.startup(server);
+ CountDownLatch connected = new CountDownLatch(1);
+ client = new ZooKeeper("127.0.0.1:" + port, 10000, e -> {
+ if (e.getState() == KeeperState.SyncConnected) {
+ connected.countDown();
+ }
+ });
+ assertTrue(connected.await(20, TimeUnit.SECONDS), "the test client did
not connect");
+ client.create(NODE, bytes("one"), Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
+ }
+
+ @Override
+ protected void doPostTearDown() throws Exception {
+ if (client != null) {
+ client.close();
+ }
+ if (factory != null) {
+ factory.shutdown();
+ }
+ if (server != null) {
+ server.shutdown();
+ }
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() {
+ return new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("zookeeper://127.0.0.1:" + port + NODE +
"?repeat=true&backoff=200").routeId("watch")
+ .to("mock:data");
+ }
+ };
+ }
+
+ @Test
+ void consumesAgainAfterTheConsumerWasStoppedAndStarted() throws Exception {
+ MockEndpoint mock = getMockEndpoint("mock:data");
+ await().atMost(20, TimeUnit.SECONDS).until(() -> received(mock,
"one"));
+
+ Consumer consumer = context.getRoute("watch").getConsumer();
+ ServiceHelper.stopService(consumer);
+ ServiceHelper.startService(consumer);
+
+ // the started consumer reads the node again
+ await().atMost(20, TimeUnit.SECONDS).untilAsserted(() ->
assertTrue(count(mock, "one") >= 2,
+ "the consumer did not read the node after it was started
again"));
+ // and the thread of the first start has ended
+ await().atMost(20, TimeUnit.SECONDS).untilAsserted(() ->
assertTrue(operationsThreads() == 1,
+ "the consumer must have one operations thread, not " +
operationsThreads()));
+
+ // update through the consumer's own session once it waits for a
change, so the change cannot be missed
+ ZooKeeperEndpoint endpoint = (ZooKeeperEndpoint)
context.getRoute("watch").getEndpoint();
+ await().atMost(20,
TimeUnit.SECONDS).until(this::operationsThreadWaitsForAChange);
+ endpoint.getConnectionManager().getConnection().setData(NODE,
bytes("two"), -1);
+ await().atMost(20, TimeUnit.SECONDS).untilAsserted(() ->
assertTrue(received(mock, "two"),
+ "the consumer did not consume the update made after it was
started again"));
+ }
+
+ @Test
+ void stopDoesNotInterruptTheExchangeBeingRouted() throws Exception {
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicReference<Throwable> failure = new AtomicReference<>();
+ AtomicBoolean completed = new AtomicBoolean();
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("zookeeper://127.0.0.1:" + port +
NODE).routeId("blocking")
+ .process(e -> {
+ entered.countDown();
+ try {
+ completed.set(release.await(20,
TimeUnit.SECONDS));
+ } catch (InterruptedException ie) {
+ failure.set(ie);
+ Thread.currentThread().interrupt();
+ }
+ });
+ }
+ });
+ assertTrue(entered.await(20, TimeUnit.SECONDS), "the exchange was not
routed");
+
+ // a graceful stop stops the consumer first (it is not Suspendable),
then waits for the exchange
+ StatefulService consumer = (StatefulService)
context.getRoute("blocking").getConsumer();
+ CompletableFuture<Void> stop = CompletableFuture.runAsync(() -> {
+ try {
+ context.getRouteController().stopRoute("blocking", 20,
TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeCamelException(e);
+ }
+ });
+ await().atMost(20, TimeUnit.SECONDS).until(consumer::isStopped);
+
+ release.countDown();
+ stop.get(30, TimeUnit.SECONDS);
+ assertNull(failure.get(), "the exchange being routed was interrupted
by the stop");
+ assertTrue(completed.get(), "the exchange being routed did not
complete");
+ }
+
+ private long operationsThreads() {
+ // only the threads of this context, in case other tests run in the
same JVM
+ String contextName = "(" + context.getName() + ")";
+ return Thread.getAllStackTraces().keySet().stream()
+ .filter(t -> t.getName().contains("Camel-Zookeeper
OperationsExecutor"))
+ .filter(t -> t.getName().contains(contextName))
+ .filter(Thread::isAlive)
+ .count();
+ }
+
+ private boolean operationsThreadWaitsForAChange() {
+ String contextName = "(" + context.getName() + ")";
+ return Thread.getAllStackTraces().entrySet().stream()
+ .filter(e -> e.getKey().getName().contains("Camel-Zookeeper
OperationsExecutor"))
+ .filter(e -> e.getKey().getName().contains(contextName))
+ .filter(e -> e.getKey().getState() == Thread.State.WAITING)
+ .anyMatch(e -> Arrays.stream(e.getValue())
+ .anyMatch(f ->
FutureEventDrivenOperation.class.getName().equals(f.getClassName())
+ && "get".equals(f.getMethodName())));
+ }
+
+ private static boolean received(MockEndpoint mock, String body) {
+ return count(mock, body) > 0;
+ }
+
+ private static long count(MockEndpoint mock, String body) {
+ return mock.getReceivedExchanges().stream()
+ .filter(e -> body.equals(new
String(e.getIn().getBody(byte[].class), StandardCharsets.UTF_8)))
+ .count();
+ }
+
+ private static byte[] bytes(String s) {
+ return s.getBytes(StandardCharsets.UTF_8);
+ }
+}