This is an automated email from the ASF dual-hosted git repository.
martinweiler pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-kie.git
The following commit(s) were added to refs/heads/main by this push:
new df16c98e81a [incubator-kie#7269] Ensure
ProcessInstanceAtomicLockStrategy lock is acquired before starting database
transaction in embedded JobScheduler (#7271)
df16c98e81a is described below
commit df16c98e81a7c8ad0c51f2432243015389e9de73
Author: Martin Weiler <[email protected]>
AuthorDate: Mon Nov 16 11:31:38 2026 -0600
[incubator-kie#7269] Ensure ProcessInstanceAtomicLockStrategy lock is
acquired before starting database transaction in embedded JobScheduler (#7271)
* Improve order of sql statements for process instances and events
* Ensure ProcessInstanceAtomicLockStrategy lock is obtained before
transaction start in VertxJobScheduler
* Fix log statements
* Add tests
* only jobs bound to a process instance (process instance and user task
timers) are locked, process definition jobs are not
* Capture process instance lock id at timer creation instead of looking it
up at fire time
* Delete process_instances before event_types to keep row lock order
consistent
* Preserve original exception type when running timeout task under process
instance lock
* Only remove the scheduled entry for the timer that fired when the job
should not run
* Harden VertxJobSchedulerLockTest: cover all job types and timer entry
removal, remove timing hacks
---
.../kogito/persistence/jdbc/GenericRepository.java | 89 ++---
.../kie/kogito/persistence/jdbc/Repository.java | 1 +
.../ContextAwareProcessInstanceLockStrategy.java | 0
.../lock/ProcessInstanceAtomicLockStrategy.java | 0
.../impl/lock/ProcessInstanceLockStrategy.java | 0
.../process/impl/lock/WorkflowAtomicExecutor.java | 1 +
.../ProcessInstanceAtomicLockStrategyTest.java | 0
.../kie/kogito/app/jobs/impl/JobDetailsHelper.java | 10 +
.../kogito/app/jobs/impl/VertxJobScheduler.java | 41 ++-
.../jobs/integrations/JobDescriptionHelper.java | 13 +
.../app/jobs/impl/VertxJobSchedulerLockTest.java | 362 +++++++++++++++++++++
11 files changed, 472 insertions(+), 45 deletions(-)
diff --git
a/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/GenericRepository.java
b/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/GenericRepository.java
index 799f8406bf3..fc49c62bff7 100644
---
a/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/GenericRepository.java
+++
b/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/GenericRepository.java
@@ -151,18 +151,7 @@ public class GenericRepository extends Repository {
void insertInternal(String processId, String processVersion, String
rootProcessId, String rootProcessVersion,
String rootProcessInstanceId, UUID id, byte[] payload, String
businessKey, String[] eventTypes) {
try (Connection connection = dataSource.getConnection();
- PreparedStatement statement =
connection.prepareStatement(INSERT);
- PreparedStatement eventStatement =
connection.prepareStatement(DELETE_ALL_WAITING_FOR_EVENT_TYPE);
- PreparedStatement insertEventStatement =
connection.prepareStatement(INSERT_WAITING_FOR_EVENT_TYPE)) {
-
- eventStatement.setString(1, id.toString());
- eventStatement.executeUpdate();
-
- for (String eventType : eventTypes) {
- insertEventStatement.setString(1, id.toString());
- insertEventStatement.setString(2, eventType);
- insertEventStatement.executeUpdate();
- }
+ PreparedStatement statement =
connection.prepareStatement(INSERT)) {
String processInstanceId = id.toString();
statement.setString(1, processInstanceId);
@@ -181,6 +170,8 @@ public class GenericRepository extends Repository {
businessKeyStmt.executeUpdate();
}
}
+
+ updateEventTypes(connection, id, eventTypes);
} catch (Exception e) {
throw uncheckedException(e, "Error inserting process instance id:
%s, processId: %s processVersion: %s business key: %s", id, processId,
processVersion, businessKey);
}
@@ -189,18 +180,7 @@ public class GenericRepository extends Repository {
@Override
void updateInternal(String processId, String processVersion, UUID id,
byte[] payload, String[] eventTypes) {
try (Connection connection = dataSource.getConnection();
- PreparedStatement statement =
connection.prepareStatement(sqlIncludingVersion(UPDATE, processVersion));
- PreparedStatement eventStatement =
connection.prepareStatement(DELETE_ALL_WAITING_FOR_EVENT_TYPE);
- PreparedStatement insertEventStatement =
connection.prepareStatement(INSERT_WAITING_FOR_EVENT_TYPE)) {
-
- eventStatement.setString(1, id.toString());
- eventStatement.executeUpdate();
-
- for (String eventType : eventTypes) {
- insertEventStatement.setString(1, id.toString());
- insertEventStatement.setString(2, eventType);
- insertEventStatement.executeUpdate();
- }
+ PreparedStatement statement =
connection.prepareStatement(sqlIncludingVersion(UPDATE, processVersion))) {
statement.setBytes(1, payload);
statement.setString(2, processId);
@@ -209,6 +189,8 @@ public class GenericRepository extends Repository {
statement.setString(4, processVersion);
}
statement.executeUpdate();
+
+ updateEventTypes(connection, id, eventTypes);
} catch (Exception e) {
throw uncheckedException(e, "Error updating process instance %s",
id);
}
@@ -217,18 +199,7 @@ public class GenericRepository extends Repository {
@Override
boolean updateWithLock(String processId, String processVersion, UUID id,
byte[] payload, long version, String[] eventTypes) {
try (Connection connection = dataSource.getConnection();
- PreparedStatement statement =
connection.prepareStatement(sqlIncludingVersion(UPDATE_WITH_LOCK,
processVersion));
- PreparedStatement eventStatement =
connection.prepareStatement(DELETE_ALL_WAITING_FOR_EVENT_TYPE);
- PreparedStatement insertEventStatement =
connection.prepareStatement(INSERT_WAITING_FOR_EVENT_TYPE)) {
-
- eventStatement.setString(1, id.toString());
- eventStatement.executeUpdate();
-
- for (String eventType : eventTypes) {
- insertEventStatement.setString(1, id.toString());
- insertEventStatement.setString(2, eventType);
- insertEventStatement.executeUpdate();
- }
+ PreparedStatement statement =
connection.prepareStatement(sqlIncludingVersion(UPDATE_WITH_LOCK,
processVersion))) {
statement.setBytes(1, payload);
statement.setLong(2, version + 1);
@@ -239,27 +210,65 @@ public class GenericRepository extends Repository {
statement.setString(6, processVersion);
}
int count = statement.executeUpdate();
- return count == 1;
+ if (count != 1) {
+ return false;
+ }
+
+ updateEventTypes(connection, id, eventTypes);
+ return true;
} catch (Exception e) {
throw uncheckedException(e, "Error updating with lock process
instance %s", id);
}
}
+ private void updateEventTypes(Connection connection, UUID id, String[]
eventTypes) throws SQLException {
+ Set<String> newEvents = eventTypes == null || eventTypes.length == 0 ?
Collections.emptySet() : new HashSet<>(Arrays.asList(eventTypes));
+ Set<String> currentEvents = new HashSet<>();
+ try (PreparedStatement findStmt =
connection.prepareStatement(FIND_EVENT_TYPES_BY_PROCESS_INSTANCE_ID)) {
+ findStmt.setString(1, id.toString());
+ try (ResultSet rs = findStmt.executeQuery()) {
+ while (rs.next()) {
+ currentEvents.add(rs.getString(1));
+ }
+ }
+ }
+
+ if (currentEvents.equals(newEvents)) {
+ LOGGER.debug("Event types unchanged for process instance id: {},
skipping update", id);
+ return;
+ }
+
+ LOGGER.debug("Updating event types for process instance id: {}
(current: {}, new: {})", id, currentEvents, newEvents);
+ try (PreparedStatement deleteStmt =
connection.prepareStatement(DELETE_ALL_WAITING_FOR_EVENT_TYPE);
+ PreparedStatement insertStmt =
connection.prepareStatement(INSERT_WAITING_FOR_EVENT_TYPE)) {
+ deleteStmt.setString(1, id.toString());
+ deleteStmt.executeUpdate();
+
+ for (String eventType : newEvents) {
+ LOGGER.trace("Inserting event type: {} for process instance
id: {}", eventType, id);
+ insertStmt.setString(1, id.toString());
+ insertStmt.setString(2, eventType);
+ insertStmt.executeUpdate();
+ }
+ }
+ }
+
@Override
boolean deleteInternal(String processId, String processVersion, UUID id) {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement =
connection.prepareStatement(sqlIncludingVersion(DELETE, processVersion));
PreparedStatement eventStatement =
connection.prepareStatement(DELETE_ALL_WAITING_FOR_EVENT_TYPE)) {
- eventStatement.setString(1, id.toString());
- eventStatement.executeUpdate();
-
+ // process_instances first, then event_types: same lock order as
insert/update, to avoid deadlocks between concurrent writers
statement.setString(1, processId);
statement.setString(2, id.toString());
if (processVersion != null) {
statement.setString(3, processVersion);
}
int count = statement.executeUpdate();
+
+ eventStatement.setString(1, id.toString());
+ eventStatement.executeUpdate();
return count == 1;
} catch (Exception e) {
throw uncheckedException(e, "Error deleting process instance %s",
id);
diff --git
a/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/Repository.java
b/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/Repository.java
index 6cdf8e9888c..66b316f435f 100644
---
a/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/Repository.java
+++
b/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/Repository.java
@@ -48,6 +48,7 @@ abstract class Repository {
static final String ROOT_PROCESS_VERSION_IS_NULL = "and
root_process_version is null";
static final String FIND_ALL_WAITING_FOR_EVENT_TYPE =
"SELECT process_instances.id, payload, process_id,
process_version, root_process_id, root_process_version, version FROM
event_types, process_instances WHERE process_instances.id =
event_types.process_instance_id AND process_id = ? AND event_type = ?";
+ static final String FIND_EVENT_TYPES_BY_PROCESS_INSTANCE_ID = "SELECT
event_type FROM event_types WHERE process_instance_id = ?";
static final String DELETE_ALL_WAITING_FOR_EVENT_TYPE = "DELETE FROM
event_types WHERE process_instance_id = ?";
static final String INSERT_WAITING_FOR_EVENT_TYPE = "INSERT INTO
event_types (process_instance_id, event_type) VALUES(?,?)";
diff --git
a/kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ContextAwareProcessInstanceLockStrategy.java
b/kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ContextAwareProcessInstanceLockStrategy.java
similarity index 100%
rename from
kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ContextAwareProcessInstanceLockStrategy.java
rename to
kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ContextAwareProcessInstanceLockStrategy.java
diff --git
a/kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategy.java
b/kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategy.java
similarity index 100%
rename from
kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategy.java
rename to
kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategy.java
diff --git
a/kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceLockStrategy.java
b/kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceLockStrategy.java
similarity index 100%
rename from
kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceLockStrategy.java
rename to
kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceLockStrategy.java
diff --git
a/kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
b/kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
similarity index 97%
rename from
kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
rename to
kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
index f875e1ef977..e5df7885bcb 100644
---
a/kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
+++
b/kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
@@ -18,6 +18,7 @@
*/
package org.kie.kogito.process.impl.lock;
+@FunctionalInterface
public interface WorkflowAtomicExecutor<T> {
T execute();
diff --git
a/kogito-jbpm/jbpm-flow/src/test/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategyTest.java
b/kogito-api/kogito-services/src/test/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategyTest.java
similarity index 100%
rename from
kogito-jbpm/jbpm-flow/src/test/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategyTest.java
rename to
kogito-api/kogito-services/src/test/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategyTest.java
diff --git
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/JobDetailsHelper.java
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/JobDetailsHelper.java
index 1b33eb91ec9..e8838d236fe 100644
---
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/JobDetailsHelper.java
+++
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/JobDetailsHelper.java
@@ -45,6 +45,16 @@ public final class JobDetailsHelper {
return jobDescription;
}
+ /**
+ * Resolves the process instance id the job belongs to, or null when the
job is not bound to a process instance.
+ */
+ public static String resolveProcessInstanceId(JobDetails jobDetails) {
+ if (jobDetails.getRecipient() == null) {
+ return null;
+ }
+ return
JobDescriptionHelper.resolveProcessInstanceId(extractJobDescription(jobDetails));
+ }
+
public static JobDetails newScheduledJobDetails(JobDescription
jobDescription) {
Long delay =
Optional.ofNullable(jobDescription.expirationTime().repeatInterval()).orElse(0L);
diff --git
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/VertxJobScheduler.java
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/VertxJobScheduler.java
index 0b3bcb9bfcc..3715717931a 100644
---
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/VertxJobScheduler.java
+++
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/VertxJobScheduler.java
@@ -30,6 +30,7 @@ import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
import org.kie.kogito.app.jobs.api.JobDescriptionMerger;
import org.kie.kogito.app.jobs.api.JobDetailsEventAdapter;
@@ -59,6 +60,7 @@ import
org.kie.kogito.jobs.service.model.JobExecutionExceptionDetails;
import org.kie.kogito.jobs.service.model.JobStatus;
import org.kie.kogito.jobs.service.model.RecipientInstance;
import org.kie.kogito.jobs.service.utils.DateUtil;
+import org.kie.kogito.process.impl.lock.ProcessInstanceAtomicLockStrategy;
import org.kie.kogito.timer.Trigger;
import org.kie.kogito.timer.impl.SimpleTimerTrigger;
import org.slf4j.Logger;
@@ -418,12 +420,16 @@ public class VertxJobScheduler implements JobScheduler,
Handler<Long> {
reconcileScheduling(null, jobContext, cancelledJobDetails);
}
- private void timeout(Long timerId, String jobId) {
+ private void timeout(Long timerId, String mapKey, String jobId, String
processInstanceId) {
LOG.debug("Executing timeout with timer Id {} and jobId {}", timerId,
jobId);
- workerExecutor.executeBlocking(newTimeoutTask(timerId, jobId), false);
+ workerExecutor.executeBlocking(newTimeoutTask(timerId, mapKey, jobId,
processInstanceId), false);
}
- private Callable<JobTimeoutExecution> newTimeoutTask(Long timerId, String
jobId) {
+ /**
+ * @param processInstanceId the process instance the job belongs to,
resolved when the timer was created (not at fire time,
+ * as the scheduled entry may have been removed in the meantime),
or null if the job is not bound to a process instance
+ */
+ private Callable<JobTimeoutExecution> newTimeoutTask(Long timerId, String
mapKey, String jobId, String processInstanceId) {
Callable<JobTimeoutExecution> current = new
Callable<JobTimeoutExecution>() {
@Override
public JobTimeoutExecution call() throws Exception {
@@ -434,7 +440,8 @@ public class VertxJobScheduler implements JobScheduler,
Handler<Long> {
boolean shouldRun = jobStore.shouldRun(jobContext, jobId);
if (!shouldRun) {
LOG.trace("Timeout {} with jobId {} won't run", timerId,
jobId);
- VertxJobScheduler.this.jobsScheduled.remove(jobId);
+ // only drop the entry if it still belongs to the timer
that fired, it may have been rescheduled under the same key
+
VertxJobScheduler.this.jobsScheduled.computeIfPresent(mapKey, (key, timerInfo)
-> timerId.equals(timerInfo.timerId()) ? null : timerInfo);
return null;
}
@@ -471,6 +478,28 @@ public class VertxJobScheduler implements JobScheduler,
Handler<Long> {
for (JobTimeoutInterceptor interceptor : interceptors) {
current = interceptor.chainIntercept(current);
}
+
+ // only jobs bound to a process instance (process instance and user
task timers) are locked, process definition jobs are not
+ if (processInstanceId != null) {
+ final Callable<JobTimeoutExecution> intercepted = current;
+ return () -> {
+ // the lock executor cannot throw checked exceptions, so the
failure is captured and rethrown as is once the lock is released
+ AtomicReference<Exception> failure = new AtomicReference<>();
+ JobTimeoutExecution execution =
ProcessInstanceAtomicLockStrategy.instance().executeOperation(processInstanceId,
() -> {
+ try {
+ return intercepted.call();
+ } catch (Exception e) {
+ failure.set(e);
+ return null;
+ }
+ });
+ if (failure.get() != null) {
+ throw failure.get();
+ }
+ return execution;
+ };
+ }
+
return current;
}
@@ -588,10 +617,12 @@ public class VertxJobScheduler implements JobScheduler,
Handler<Long> {
ZonedDateTime now = DateUtil.now();
Long diff = ChronoUnit.MILLIS.between(now, trigger);
Long delay = Math.max(1, diff);
+ String mapKey = getMapKey(jobDetails);
+ String processInstanceId =
JobDetailsHelper.resolveProcessInstanceId(jobDetails);
Long timerId = this.vertx.setTimer(delay, new Handler<Long>() {
@Override
public void handle(Long timerId) {
- timeout(timerId, jobDetails.getId());
+ timeout(timerId, mapKey, jobDetails.getId(),
processInstanceId);
}
});
return new TimerInfo(jobDetails.getId(), jobDetails.getRetries(),
timerId, jobDetails.getTrigger().hasNextFireTime());
diff --git
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/integrations/JobDescriptionHelper.java
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/integrations/JobDescriptionHelper.java
index 955e4cadd0d..84d3fdabc72 100644
---
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/integrations/JobDescriptionHelper.java
+++
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/integrations/JobDescriptionHelper.java
@@ -63,6 +63,19 @@ public class JobDescriptionHelper {
return correlationId;
}
+ /**
+ * Resolves the id of the process instance that owns the job, if any.
Process definition jobs
+ * (e.g. start timers) are not bound to a process instance and return null.
+ */
+ public static <T extends JobDescription> String resolveProcessInstanceId(T
jobDescription) {
+ if (jobDescription instanceof ProcessInstanceJobDescription
processInstanceJobDescription) {
+ return processInstanceJobDescription.processInstanceId();
+ } else if (jobDescription instanceof UserTaskInstanceJobDescription
userTaskInstanceJobDescription) {
+ return userTaskInstanceJobDescription.processInstanceId();
+ }
+ return null;
+ }
+
public static <T extends JobDescription> String resolveProcessId(T
jobDescription) {
String processId = null;
if (jobDescription instanceof ProcessJobDescription
processJobDescription) {
diff --git
a/kogito-jobs/jobs-common-embedded/src/test/java/org/kie/kogito/app/jobs/impl/VertxJobSchedulerLockTest.java
b/kogito-jobs/jobs-common-embedded/src/test/java/org/kie/kogito/app/jobs/impl/VertxJobSchedulerLockTest.java
new file mode 100644
index 00000000000..a5ec43b08c2
--- /dev/null
+++
b/kogito-jobs/jobs-common-embedded/src/test/java/org/kie/kogito/app/jobs/impl/VertxJobSchedulerLockTest.java
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.kie.kogito.app.jobs.impl;
+
+import java.lang.reflect.Field;
+import java.time.Duration;
+import java.time.ZonedDateTime;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.kie.kogito.app.jobs.api.JobDescriptionMerger;
+import org.kie.kogito.app.jobs.api.JobDetailsEventAdapter;
+import org.kie.kogito.app.jobs.api.JobExecutor;
+import org.kie.kogito.app.jobs.api.JobScheduler;
+import org.kie.kogito.app.jobs.api.JobSchedulerBuilder;
+import org.kie.kogito.app.jobs.api.JobTimeoutInterceptor;
+import
org.kie.kogito.app.jobs.integrations.ProcessInstanceJobDescriptionJobInstanceEventAdapter;
+import
org.kie.kogito.app.jobs.integrations.ProcessInstanceJobDescriptionMerger;
+import
org.kie.kogito.app.jobs.integrations.ProcessJobDescriptionJobInstanceEventAdapter;
+import org.kie.kogito.app.jobs.integrations.ProcessJobDescriptionMerger;
+import
org.kie.kogito.app.jobs.integrations.UserTaskInstanceJobDescriptionJobInstanceEventAdapter;
+import
org.kie.kogito.app.jobs.integrations.UserTaskInstanceJobDescriptorMerger;
+import org.kie.kogito.app.jobs.spi.JobContext;
+import org.kie.kogito.app.jobs.spi.JobStore;
+import org.kie.kogito.app.jobs.spi.memory.MemoryJobContextFactory;
+import org.kie.kogito.app.jobs.spi.memory.MemoryJobStore;
+import org.kie.kogito.jobs.ExactExpirationTime;
+import org.kie.kogito.jobs.JobDescription;
+import org.kie.kogito.jobs.descriptors.ProcessInstanceJobDescription;
+import org.kie.kogito.jobs.descriptors.ProcessJobDescription;
+import org.kie.kogito.jobs.descriptors.UserTaskInstanceJobDescription;
+import org.kie.kogito.jobs.service.model.JobDetails;
+import org.kie.kogito.process.impl.lock.ProcessInstanceAtomicLockStrategy;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+
+class VertxJobSchedulerLockTest {
+
+ private static final String SERVICE_URL = "http://localhost:8080";
+
+ private JobScheduler jobScheduler;
+
+ @AfterEach
+ void closeScheduler() {
+ if (jobScheduler != null) {
+ jobScheduler.close();
+ jobScheduler = null;
+ }
+ }
+
+ /**
+ * Records what the Vert.x worker thread sees when a transaction-like
interceptor starts.
+ */
+ private static class LockObservation {
+ final AtomicBoolean executed = new AtomicBoolean();
+ final AtomicBoolean lockedKey = new AtomicBoolean();
+ final AtomicBoolean lockedOtherKey = new AtomicBoolean();
+ }
+
+ private JobScheduler newScheduler(JobStore jobStore,
JobDetailsEventAdapter adapter, JobDescriptionMerger merger,
+ LatchExecutionJobSchedulerListener listener, JobExecutor executor,
JobTimeoutInterceptor... interceptors) {
+ jobScheduler = JobSchedulerBuilder.newJobSchedulerBuilder()
+ .withJobExecutors(executor)
+ .withJobEventAdapters(adapter)
+ .withEventPublishers(new TestEventPublisher())
+ .withJobContextFactory(new MemoryJobContextFactory())
+ .withJobStore(jobStore)
+ .withJobSchedulerListeners(listener)
+ .withJobDescriptorMergers(merger)
+ .withTimeoutInterceptor(interceptors)
+ .build();
+ jobScheduler.init();
+ return jobScheduler;
+ }
+
+ /**
+ * Schedules the job and observes, from the worker thread and at the point
where a transaction interceptor would start,
+ * whether the lock for {@code key} and for {@code otherKey} is held. Then
verifies the lock for {@code key} was released.
+ */
+ private LockObservation runAndObserveLocks(JobDescription jobDescription,
JobDetailsEventAdapter adapter, JobDescriptionMerger merger,
+ String key, String otherKey) throws Exception {
+ LockObservation observation = new LockObservation();
+ ProcessInstanceAtomicLockStrategy locks =
ProcessInstanceAtomicLockStrategy.instance();
+
+ JobTimeoutInterceptor observingInterceptor = callable -> () -> {
+ observation.lockedKey.set(locks.isLockedByCurrentThread(key));
+
observation.lockedOtherKey.set(locks.isLockedByCurrentThread(otherKey));
+ return callable.call();
+ };
+
+ LatchExecutionJobSchedulerListener listener = new
LatchExecutionJobSchedulerListener();
+ TestJobExecutor executor = new TestJobExecutor();
+ newScheduler(new MemoryJobStore(), adapter, merger, listener,
executor, observingInterceptor);
+
+ jobScheduler.schedule(jobDescription);
+ listener.waitForExecution(3000L);
+ observation.executed.set(listener.isExecuted() &&
executor.getJobsExecuted().contains(jobDescription.id()));
+
+ // the lock must not leak: it can be acquired from another thread once
the timeout task finished
+ assertTimeoutPreemptively(Duration.ofSeconds(3), () ->
locks.executeOperation(key, () -> null));
+ return observation;
+ }
+
+ private static ProcessInstanceJobDescription processInstanceJob(String
jobId, String processInstanceId, ZonedDateTime when) {
+ return
ProcessInstanceJobDescription.newProcessInstanceJobDescriptionBuilder()
+ .id(jobId)
+ .timerId("timer-" + jobId)
+ .processId("testProcess")
+ .processInstanceId(processInstanceId)
+ .expirationTime(ExactExpirationTime.of(when))
+ .build();
+ }
+
+ /**
+ * The lock for the process instance must already be held by the worker
thread when the transaction interceptor starts.
+ */
+ @Test
+ void testProcessInstanceJobLocksBeforeInterceptorRuns() throws Exception {
+ final String jobId = "lock-order-job-1";
+ final String processInstanceId = "pi-lock-order-123";
+
+ LockObservation observation = runAndObserveLocks(
+ processInstanceJob(jobId, processInstanceId,
ZonedDateTime.now().plus(Duration.ofMillis(50))),
+ new
ProcessInstanceJobDescriptionJobInstanceEventAdapter(SERVICE_URL),
+ new ProcessInstanceJobDescriptionMerger(),
+ processInstanceId, jobId);
+
+ assertThat(observation.executed).isTrue();
+ assertThat(observation.lockedKey).as("process instance lock must be
held when the transaction interceptor starts").isTrue();
+ assertThat(observation.lockedOtherKey).as("job id must not be used as
lock key").isFalse();
+ }
+
+ /**
+ * User task timers must lock on the owning process instance id, not on
the job id
+ * (which is what the job correlationId holds for this kind of job).
+ */
+ @Test
+ void testUserTaskJobLocksOnProcessInstanceId() throws Exception {
+ final String jobId = "usertask-job-4";
+ final String processInstanceId = "pi-usertask-789";
+
+ UserTaskInstanceJobDescription jobDescription =
UserTaskInstanceJobDescription.newUserTaskInstanceJobDescriptionBuilder()
+ .id(jobId)
+ .userTaskInstanceId("ut-1")
+ .metadata(new HashMap<>())
+ .processId("testProcess")
+ .processInstanceId(processInstanceId)
+
.expirationTime(ExactExpirationTime.of(ZonedDateTime.now().plus(Duration.ofMillis(50))))
+ .build();
+
+ LockObservation observation = runAndObserveLocks(jobDescription,
+ new
UserTaskInstanceJobDescriptionJobInstanceEventAdapter(SERVICE_URL),
+ new UserTaskInstanceJobDescriptorMerger(),
+ processInstanceId, jobId);
+
+ assertThat(observation.executed).isTrue();
+ assertThat(observation.lockedKey).as("process instance lock must be
held for user task jobs").isTrue();
+ assertThat(observation.lockedOtherKey).as("job id must not be used as
lock key").isFalse();
+ }
+
+ /**
+ * Process definition timers (e.g. start timers) are not bound to any
process instance,
+ * so no lock must be taken, in particular not on the process id.
+ */
+ @Test
+ void testProcessDefinitionJobIsNotLocked() throws Exception {
+ final String processId = "testProcessDefinition";
+
+ ProcessJobDescription jobDescription = ProcessJobDescription.of(
+
ExactExpirationTime.of(ZonedDateTime.now().plus(Duration.ofMillis(50))),
processId, "1.0");
+
+ LockObservation observation = runAndObserveLocks(jobDescription,
+ new ProcessJobDescriptionJobInstanceEventAdapter(SERVICE_URL),
+ new ProcessJobDescriptionMerger(),
+ processId, jobDescription.id());
+
+ assertThat(observation.executed).isTrue();
+ assertThat(observation.lockedKey).as("process definition jobs must not
lock on the process id").isFalse();
+ assertThat(observation.lockedOtherKey).as("process definition jobs
must not lock on the job id").isFalse();
+ }
+
+ /**
+ * Jobs that are not related to processes at all must not take any lock.
+ */
+ @Test
+ void testJobWithoutProcessInstanceIsNotLocked() throws Exception {
+ final String jobId = "no-process-job-3";
+
+ LockObservation observation = runAndObserveLocks(
+ new TestJobDescription(jobId,
ZonedDateTime.now().plus(Duration.ofMillis(50))),
+ new TestJobDetailsEventAdapter(),
+ new TestJobDescriptionMerger(),
+ jobId, jobId);
+
+ assertThat(observation.executed).isTrue();
+ assertThat(observation.lockedKey).as("jobs without a process instance
must not be locked").isFalse();
+ }
+
+ /**
+ * While another thread holds the process instance lock (e.g. an incoming
REST request), the timer thread must wait for it
+ * BEFORE the transaction interceptor starts, so the transaction observes
the latest committed state rather than a stale snapshot.
+ */
+ @Test
+ void testConcurrentLockContentionGuaranteesFreshTransactionSnapshot()
throws Exception {
+ final String jobId = "concurrent-lock-job-2";
+ final String processInstanceId = "pi-concurrent-456";
+
+ final AtomicInteger simulatedDbVersion = new AtomicInteger(1);
+ final AtomicInteger versionObservedAtTxStart = new AtomicInteger(-1);
+ final CountDownLatch jobExecuted = new CountDownLatch(1);
+ // the periodic job store sync goes through the interceptors too, so
the job itself is what tells us which transaction is relevant
+ final ThreadLocal<Integer> versionAtCurrentTxStart = new
ThreadLocal<>();
+
+ JobTimeoutInterceptor mockTransactionInterceptor = callable -> () -> {
+ // the DB version observed at the exact moment the transaction
starts
+ versionAtCurrentTxStart.set(simulatedDbVersion.get());
+ try {
+ return callable.call();
+ } finally {
+ versionAtCurrentTxStart.remove();
+ }
+ };
+
+ JobExecutor executor = new JobExecutor() {
+ @Override
+ public boolean accept(JobDetails jobDetails) {
+ return true;
+ }
+
+ @Override
+ public void execute(JobDetails jobDetails) {
+ versionObservedAtTxStart.set(versionAtCurrentTxStart.get());
+ jobExecuted.countDown();
+ }
+ };
+
+ LatchExecutionJobSchedulerListener listener = new
LatchExecutionJobSchedulerListener();
+ newScheduler(new MemoryJobStore(), new
ProcessInstanceJobDescriptionJobInstanceEventAdapter(SERVICE_URL),
+ new ProcessInstanceJobDescriptionMerger(), listener, executor,
mockTransactionInterceptor);
+
+ CountDownLatch externalLockAcquired = new CountDownLatch(1);
+ CountDownLatch externalRelease = new CountDownLatch(1);
+
+ ExecutorService backgroundExecutor =
Executors.newSingleThreadExecutor();
+ try {
+ Future<?> external = backgroundExecutor.submit(() ->
ProcessInstanceAtomicLockStrategy.instance().executeOperation(processInstanceId,
() -> {
+ externalLockAcquired.countDown();
+ try {
+ externalRelease.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ // update the state/version before releasing the lock
+ simulatedDbVersion.set(2);
+ return null;
+ }));
+
+ assertThat(externalLockAcquired.await(2,
TimeUnit.SECONDS)).isTrue();
+
+ // the timer fires while the external thread still holds the lock
+ jobScheduler.schedule(processInstanceJob(jobId, processInstanceId,
ZonedDateTime.now().plus(Duration.ofMillis(10))));
+
+ // the job (and so its transaction) must not run while the lock is
held by someone else
+ assertThat(jobExecuted.await(300, TimeUnit.MILLISECONDS))
+ .as("job must not run while the process instance lock is
held by another thread")
+ .isFalse();
+
+ externalRelease.countDown();
+ external.get(3, TimeUnit.SECONDS);
+ } finally {
+ externalRelease.countDown();
+ backgroundExecutor.shutdownNow();
+ }
+
+ assertThat(jobExecuted.await(3, TimeUnit.SECONDS)).isTrue();
+ assertThat(versionObservedAtTxStart.get())
+ .as("transaction must start AFTER the lock is acquired,
observing the latest state (version 2)")
+ .isEqualTo(2);
+ }
+
+ /**
+ * A timer that finds its job should not run (e.g. it was rescheduled in
the meantime) must only drop its own entry
+ * from the scheduled jobs, not the entry of the timer that replaced it
under the same key.
+ */
+ @Test
+ void testShouldNotRunDoesNotRemoveEntryOfRescheduledTimer() throws
Exception {
+ final String jobId = "reschedule-job-5";
+ final String processInstanceId = "pi-reschedule-321";
+
+ CountDownLatch shouldRunEntered = new CountDownLatch(1);
+ CountDownLatch shouldRunRelease = new CountDownLatch(1);
+ AtomicBoolean firstCall = new AtomicBoolean(true);
+
+ // the first timeout is held inside shouldRun and then told not to
run, later calls behave normally
+ JobStore blockingStore = new MemoryJobStore() {
+ @Override
+ public boolean shouldRun(JobContext jobContext, String id) {
+ if (firstCall.compareAndSet(true, false)) {
+ shouldRunEntered.countDown();
+ try {
+ shouldRunRelease.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ return false;
+ }
+ return super.shouldRun(jobContext, id);
+ }
+ };
+
+ newScheduler(blockingStore, new
ProcessInstanceJobDescriptionJobInstanceEventAdapter(SERVICE_URL),
+ new ProcessInstanceJobDescriptionMerger(), new
LatchExecutionJobSchedulerListener(), new TestJobExecutor());
+
+ jobScheduler.schedule(processInstanceJob(jobId, processInstanceId,
ZonedDateTime.now().plus(Duration.ofMillis(50))));
+ assertThat(shouldRunEntered.await(3, TimeUnit.SECONDS)).isTrue();
+
+ // while the first timer is still in flight, the job is rescheduled
and gets a new timer under the same key
+ jobScheduler.reschedule(processInstanceJob(jobId, processInstanceId,
ZonedDateTime.now().plus(Duration.ofMinutes(10))));
+ shouldRunRelease.countDown();
+
+ Map<String, ?> jobsScheduled = jobsScheduled();
+ String mapKey = jobId + "-0";
+ assertThat(jobsScheduled).containsKey(mapKey);
+ // the first timeout has now completed its "should not run" handling,
the entry of the rescheduled timer must survive
+ await().during(500, TimeUnit.MILLISECONDS).atMost(2, TimeUnit.SECONDS)
+ .untilAsserted(() ->
assertThat(jobsScheduled).containsKey(mapKey));
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, ?> jobsScheduled() throws Exception {
+ Field field =
VertxJobScheduler.class.getDeclaredField("jobsScheduled");
+ field.setAccessible(true);
+ return (Map<String, ?>) field.get(jobScheduler);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]