This is an automated email from the ASF dual-hosted git repository.

martinweiler pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-kie.git


The following commit(s) were added to refs/heads/main by this push:
     new df16c98e81a [incubator-kie#7269] Ensure 
ProcessInstanceAtomicLockStrategy lock is acquired before starting database 
transaction in embedded JobScheduler (#7271)
df16c98e81a is described below

commit df16c98e81a7c8ad0c51f2432243015389e9de73
Author: Martin Weiler <[email protected]>
AuthorDate: Mon Nov 16 11:31:38 2026 -0600

    [incubator-kie#7269] Ensure ProcessInstanceAtomicLockStrategy lock is 
acquired before starting database transaction in embedded JobScheduler (#7271)
    
    * Improve order of sql statements for process instances and events
    
    * Ensure ProcessInstanceAtomicLockStrategy lock is obtained before 
transaction start in VertxJobScheduler
    
    * Fix log statements
    
    * Add tests
    
    * only jobs bound to a process instance (process instance and user task 
timers) are locked, process definition jobs are not
    
    * Capture process instance lock id at timer creation instead of looking it 
up at fire time
    
    * Delete process_instances before event_types to keep row lock order 
consistent
    
    * Preserve original exception type when running timeout task under process 
instance lock
    
    * Only remove the scheduled entry for the timer that fired when the job 
should not run
    
    * Harden VertxJobSchedulerLockTest: cover all job types and timer entry 
removal, remove timing hacks
---
 .../kogito/persistence/jdbc/GenericRepository.java |  89 ++---
 .../kie/kogito/persistence/jdbc/Repository.java    |   1 +
 .../ContextAwareProcessInstanceLockStrategy.java   |   0
 .../lock/ProcessInstanceAtomicLockStrategy.java    |   0
 .../impl/lock/ProcessInstanceLockStrategy.java     |   0
 .../process/impl/lock/WorkflowAtomicExecutor.java  |   1 +
 .../ProcessInstanceAtomicLockStrategyTest.java     |   0
 .../kie/kogito/app/jobs/impl/JobDetailsHelper.java |  10 +
 .../kogito/app/jobs/impl/VertxJobScheduler.java    |  41 ++-
 .../jobs/integrations/JobDescriptionHelper.java    |  13 +
 .../app/jobs/impl/VertxJobSchedulerLockTest.java   | 362 +++++++++++++++++++++
 11 files changed, 472 insertions(+), 45 deletions(-)

diff --git 
a/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/GenericRepository.java
 
b/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/GenericRepository.java
index 799f8406bf3..fc49c62bff7 100644
--- 
a/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/GenericRepository.java
+++ 
b/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/GenericRepository.java
@@ -151,18 +151,7 @@ public class GenericRepository extends Repository {
     void insertInternal(String processId, String processVersion, String 
rootProcessId, String rootProcessVersion,
             String rootProcessInstanceId, UUID id, byte[] payload, String 
businessKey, String[] eventTypes) {
         try (Connection connection = dataSource.getConnection();
-                PreparedStatement statement = 
connection.prepareStatement(INSERT);
-                PreparedStatement eventStatement = 
connection.prepareStatement(DELETE_ALL_WAITING_FOR_EVENT_TYPE);
-                PreparedStatement insertEventStatement = 
connection.prepareStatement(INSERT_WAITING_FOR_EVENT_TYPE)) {
-
-            eventStatement.setString(1, id.toString());
-            eventStatement.executeUpdate();
-
-            for (String eventType : eventTypes) {
-                insertEventStatement.setString(1, id.toString());
-                insertEventStatement.setString(2, eventType);
-                insertEventStatement.executeUpdate();
-            }
+                PreparedStatement statement = 
connection.prepareStatement(INSERT)) {
 
             String processInstanceId = id.toString();
             statement.setString(1, processInstanceId);
@@ -181,6 +170,8 @@ public class GenericRepository extends Repository {
                     businessKeyStmt.executeUpdate();
                 }
             }
+
+            updateEventTypes(connection, id, eventTypes);
         } catch (Exception e) {
             throw uncheckedException(e, "Error inserting process instance id: 
%s, processId: %s processVersion: %s business key: %s", id, processId, 
processVersion, businessKey);
         }
@@ -189,18 +180,7 @@ public class GenericRepository extends Repository {
     @Override
     void updateInternal(String processId, String processVersion, UUID id, 
byte[] payload, String[] eventTypes) {
         try (Connection connection = dataSource.getConnection();
-                PreparedStatement statement = 
connection.prepareStatement(sqlIncludingVersion(UPDATE, processVersion));
-                PreparedStatement eventStatement = 
connection.prepareStatement(DELETE_ALL_WAITING_FOR_EVENT_TYPE);
-                PreparedStatement insertEventStatement = 
connection.prepareStatement(INSERT_WAITING_FOR_EVENT_TYPE)) {
-
-            eventStatement.setString(1, id.toString());
-            eventStatement.executeUpdate();
-
-            for (String eventType : eventTypes) {
-                insertEventStatement.setString(1, id.toString());
-                insertEventStatement.setString(2, eventType);
-                insertEventStatement.executeUpdate();
-            }
+                PreparedStatement statement = 
connection.prepareStatement(sqlIncludingVersion(UPDATE, processVersion))) {
 
             statement.setBytes(1, payload);
             statement.setString(2, processId);
@@ -209,6 +189,8 @@ public class GenericRepository extends Repository {
                 statement.setString(4, processVersion);
             }
             statement.executeUpdate();
+
+            updateEventTypes(connection, id, eventTypes);
         } catch (Exception e) {
             throw uncheckedException(e, "Error updating process instance %s", 
id);
         }
@@ -217,18 +199,7 @@ public class GenericRepository extends Repository {
     @Override
     boolean updateWithLock(String processId, String processVersion, UUID id, 
byte[] payload, long version, String[] eventTypes) {
         try (Connection connection = dataSource.getConnection();
-                PreparedStatement statement = 
connection.prepareStatement(sqlIncludingVersion(UPDATE_WITH_LOCK, 
processVersion));
-                PreparedStatement eventStatement = 
connection.prepareStatement(DELETE_ALL_WAITING_FOR_EVENT_TYPE);
-                PreparedStatement insertEventStatement = 
connection.prepareStatement(INSERT_WAITING_FOR_EVENT_TYPE)) {
-
-            eventStatement.setString(1, id.toString());
-            eventStatement.executeUpdate();
-
-            for (String eventType : eventTypes) {
-                insertEventStatement.setString(1, id.toString());
-                insertEventStatement.setString(2, eventType);
-                insertEventStatement.executeUpdate();
-            }
+                PreparedStatement statement = 
connection.prepareStatement(sqlIncludingVersion(UPDATE_WITH_LOCK, 
processVersion))) {
 
             statement.setBytes(1, payload);
             statement.setLong(2, version + 1);
@@ -239,27 +210,65 @@ public class GenericRepository extends Repository {
                 statement.setString(6, processVersion);
             }
             int count = statement.executeUpdate();
-            return count == 1;
+            if (count != 1) {
+                return false;
+            }
+
+            updateEventTypes(connection, id, eventTypes);
+            return true;
         } catch (Exception e) {
             throw uncheckedException(e, "Error updating with lock process 
instance %s", id);
         }
     }
 
+    private void updateEventTypes(Connection connection, UUID id, String[] 
eventTypes) throws SQLException {
+        Set<String> newEvents = eventTypes == null || eventTypes.length == 0 ? 
Collections.emptySet() : new HashSet<>(Arrays.asList(eventTypes));
+        Set<String> currentEvents = new HashSet<>();
+        try (PreparedStatement findStmt = 
connection.prepareStatement(FIND_EVENT_TYPES_BY_PROCESS_INSTANCE_ID)) {
+            findStmt.setString(1, id.toString());
+            try (ResultSet rs = findStmt.executeQuery()) {
+                while (rs.next()) {
+                    currentEvents.add(rs.getString(1));
+                }
+            }
+        }
+
+        if (currentEvents.equals(newEvents)) {
+            LOGGER.debug("Event types unchanged for process instance id: {}, 
skipping update", id);
+            return;
+        }
+
+        LOGGER.debug("Updating event types for process instance id: {} 
(current: {}, new: {})", id, currentEvents, newEvents);
+        try (PreparedStatement deleteStmt = 
connection.prepareStatement(DELETE_ALL_WAITING_FOR_EVENT_TYPE);
+                PreparedStatement insertStmt = 
connection.prepareStatement(INSERT_WAITING_FOR_EVENT_TYPE)) {
+            deleteStmt.setString(1, id.toString());
+            deleteStmt.executeUpdate();
+
+            for (String eventType : newEvents) {
+                LOGGER.trace("Inserting event type: {} for process instance 
id: {}", eventType, id);
+                insertStmt.setString(1, id.toString());
+                insertStmt.setString(2, eventType);
+                insertStmt.executeUpdate();
+            }
+        }
+    }
+
     @Override
     boolean deleteInternal(String processId, String processVersion, UUID id) {
         try (Connection connection = dataSource.getConnection();
                 PreparedStatement statement = 
connection.prepareStatement(sqlIncludingVersion(DELETE, processVersion));
                 PreparedStatement eventStatement = 
connection.prepareStatement(DELETE_ALL_WAITING_FOR_EVENT_TYPE)) {
 
-            eventStatement.setString(1, id.toString());
-            eventStatement.executeUpdate();
-
+            // process_instances first, then event_types: same lock order as 
insert/update, to avoid deadlocks between concurrent writers
             statement.setString(1, processId);
             statement.setString(2, id.toString());
             if (processVersion != null) {
                 statement.setString(3, processVersion);
             }
             int count = statement.executeUpdate();
+
+            eventStatement.setString(1, id.toString());
+            eventStatement.executeUpdate();
             return count == 1;
         } catch (Exception e) {
             throw uncheckedException(e, "Error deleting process instance %s", 
id);
diff --git 
a/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/Repository.java
 
b/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/Repository.java
index 6cdf8e9888c..66b316f435f 100644
--- 
a/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/Repository.java
+++ 
b/kogito-addons/common/persistence/jdbc/src/main/java/org/kie/kogito/persistence/jdbc/Repository.java
@@ -48,6 +48,7 @@ abstract class Repository {
     static final String ROOT_PROCESS_VERSION_IS_NULL = "and 
root_process_version is null";
     static final String FIND_ALL_WAITING_FOR_EVENT_TYPE =
             "SELECT process_instances.id, payload, process_id, 
process_version, root_process_id, root_process_version, version FROM 
event_types, process_instances WHERE process_instances.id = 
event_types.process_instance_id AND process_id = ? AND event_type = ?";
+    static final String FIND_EVENT_TYPES_BY_PROCESS_INSTANCE_ID = "SELECT 
event_type FROM event_types WHERE process_instance_id = ?";
     static final String DELETE_ALL_WAITING_FOR_EVENT_TYPE = "DELETE FROM 
event_types WHERE process_instance_id = ?";
     static final String INSERT_WAITING_FOR_EVENT_TYPE = "INSERT INTO 
event_types (process_instance_id, event_type) VALUES(?,?)";
 
diff --git 
a/kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ContextAwareProcessInstanceLockStrategy.java
 
b/kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ContextAwareProcessInstanceLockStrategy.java
similarity index 100%
rename from 
kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ContextAwareProcessInstanceLockStrategy.java
rename to 
kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ContextAwareProcessInstanceLockStrategy.java
diff --git 
a/kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategy.java
 
b/kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategy.java
similarity index 100%
rename from 
kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategy.java
rename to 
kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategy.java
diff --git 
a/kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceLockStrategy.java
 
b/kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceLockStrategy.java
similarity index 100%
rename from 
kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceLockStrategy.java
rename to 
kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/ProcessInstanceLockStrategy.java
diff --git 
a/kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
 
b/kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
similarity index 97%
rename from 
kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
rename to 
kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
index f875e1ef977..e5df7885bcb 100644
--- 
a/kogito-jbpm/jbpm-flow/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
+++ 
b/kogito-api/kogito-services/src/main/java/org/kie/kogito/process/impl/lock/WorkflowAtomicExecutor.java
@@ -18,6 +18,7 @@
  */
 package org.kie.kogito.process.impl.lock;
 
+@FunctionalInterface
 public interface WorkflowAtomicExecutor<T> {
 
     T execute();
diff --git 
a/kogito-jbpm/jbpm-flow/src/test/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategyTest.java
 
b/kogito-api/kogito-services/src/test/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategyTest.java
similarity index 100%
rename from 
kogito-jbpm/jbpm-flow/src/test/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategyTest.java
rename to 
kogito-api/kogito-services/src/test/java/org/kie/kogito/process/impl/lock/ProcessInstanceAtomicLockStrategyTest.java
diff --git 
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/JobDetailsHelper.java
 
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/JobDetailsHelper.java
index 1b33eb91ec9..e8838d236fe 100644
--- 
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/JobDetailsHelper.java
+++ 
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/JobDetailsHelper.java
@@ -45,6 +45,16 @@ public final class JobDetailsHelper {
         return jobDescription;
     }
 
+    /**
+     * Resolves the process instance id the job belongs to, or null when the 
job is not bound to a process instance.
+     */
+    public static String resolveProcessInstanceId(JobDetails jobDetails) {
+        if (jobDetails.getRecipient() == null) {
+            return null;
+        }
+        return 
JobDescriptionHelper.resolveProcessInstanceId(extractJobDescription(jobDetails));
+    }
+
     public static JobDetails newScheduledJobDetails(JobDescription 
jobDescription) {
 
         Long delay = 
Optional.ofNullable(jobDescription.expirationTime().repeatInterval()).orElse(0L);
diff --git 
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/VertxJobScheduler.java
 
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/VertxJobScheduler.java
index 0b3bcb9bfcc..3715717931a 100644
--- 
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/VertxJobScheduler.java
+++ 
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/impl/VertxJobScheduler.java
@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.kie.kogito.app.jobs.api.JobDescriptionMerger;
 import org.kie.kogito.app.jobs.api.JobDetailsEventAdapter;
@@ -59,6 +60,7 @@ import 
org.kie.kogito.jobs.service.model.JobExecutionExceptionDetails;
 import org.kie.kogito.jobs.service.model.JobStatus;
 import org.kie.kogito.jobs.service.model.RecipientInstance;
 import org.kie.kogito.jobs.service.utils.DateUtil;
+import org.kie.kogito.process.impl.lock.ProcessInstanceAtomicLockStrategy;
 import org.kie.kogito.timer.Trigger;
 import org.kie.kogito.timer.impl.SimpleTimerTrigger;
 import org.slf4j.Logger;
@@ -418,12 +420,16 @@ public class VertxJobScheduler implements JobScheduler, 
Handler<Long> {
         reconcileScheduling(null, jobContext, cancelledJobDetails);
     }
 
-    private void timeout(Long timerId, String jobId) {
+    private void timeout(Long timerId, String mapKey, String jobId, String 
processInstanceId) {
         LOG.debug("Executing timeout with timer Id {} and jobId {}", timerId, 
jobId);
-        workerExecutor.executeBlocking(newTimeoutTask(timerId, jobId), false);
+        workerExecutor.executeBlocking(newTimeoutTask(timerId, mapKey, jobId, 
processInstanceId), false);
     }
 
-    private Callable<JobTimeoutExecution> newTimeoutTask(Long timerId, String 
jobId) {
+    /**
+     * @param processInstanceId the process instance the job belongs to, 
resolved when the timer was created (not at fire time,
+     *        as the scheduled entry may have been removed in the meantime), 
or null if the job is not bound to a process instance
+     */
+    private Callable<JobTimeoutExecution> newTimeoutTask(Long timerId, String 
mapKey, String jobId, String processInstanceId) {
         Callable<JobTimeoutExecution> current = new 
Callable<JobTimeoutExecution>() {
             @Override
             public JobTimeoutExecution call() throws Exception {
@@ -434,7 +440,8 @@ public class VertxJobScheduler implements JobScheduler, 
Handler<Long> {
                 boolean shouldRun = jobStore.shouldRun(jobContext, jobId);
                 if (!shouldRun) {
                     LOG.trace("Timeout {} with jobId {} won't run", timerId, 
jobId);
-                    VertxJobScheduler.this.jobsScheduled.remove(jobId);
+                    // only drop the entry if it still belongs to the timer 
that fired, it may have been rescheduled under the same key
+                    
VertxJobScheduler.this.jobsScheduled.computeIfPresent(mapKey, (key, timerInfo) 
-> timerId.equals(timerInfo.timerId()) ? null : timerInfo);
                     return null;
                 }
 
@@ -471,6 +478,28 @@ public class VertxJobScheduler implements JobScheduler, 
Handler<Long> {
         for (JobTimeoutInterceptor interceptor : interceptors) {
             current = interceptor.chainIntercept(current);
         }
+
+        // only jobs bound to a process instance (process instance and user 
task timers) are locked, process definition jobs are not
+        if (processInstanceId != null) {
+            final Callable<JobTimeoutExecution> intercepted = current;
+            return () -> {
+                // the lock executor cannot throw checked exceptions, so the 
failure is captured and rethrown as is once the lock is released
+                AtomicReference<Exception> failure = new AtomicReference<>();
+                JobTimeoutExecution execution = 
ProcessInstanceAtomicLockStrategy.instance().executeOperation(processInstanceId,
 () -> {
+                    try {
+                        return intercepted.call();
+                    } catch (Exception e) {
+                        failure.set(e);
+                        return null;
+                    }
+                });
+                if (failure.get() != null) {
+                    throw failure.get();
+                }
+                return execution;
+            };
+        }
+
         return current;
     }
 
@@ -588,10 +617,12 @@ public class VertxJobScheduler implements JobScheduler, 
Handler<Long> {
         ZonedDateTime now = DateUtil.now();
         Long diff = ChronoUnit.MILLIS.between(now, trigger);
         Long delay = Math.max(1, diff);
+        String mapKey = getMapKey(jobDetails);
+        String processInstanceId = 
JobDetailsHelper.resolveProcessInstanceId(jobDetails);
         Long timerId = this.vertx.setTimer(delay, new Handler<Long>() {
             @Override
             public void handle(Long timerId) {
-                timeout(timerId, jobDetails.getId());
+                timeout(timerId, mapKey, jobDetails.getId(), 
processInstanceId);
             }
         });
         return new TimerInfo(jobDetails.getId(), jobDetails.getRetries(), 
timerId, jobDetails.getTrigger().hasNextFireTime());
diff --git 
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/integrations/JobDescriptionHelper.java
 
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/integrations/JobDescriptionHelper.java
index 955e4cadd0d..84d3fdabc72 100644
--- 
a/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/integrations/JobDescriptionHelper.java
+++ 
b/kogito-jobs/jobs-common-embedded/src/main/java/org/kie/kogito/app/jobs/integrations/JobDescriptionHelper.java
@@ -63,6 +63,19 @@ public class JobDescriptionHelper {
         return correlationId;
     }
 
+    /**
+     * Resolves the id of the process instance that owns the job, if any. 
Process definition jobs
+     * (e.g. start timers) are not bound to a process instance and return null.
+     */
+    public static <T extends JobDescription> String resolveProcessInstanceId(T 
jobDescription) {
+        if (jobDescription instanceof ProcessInstanceJobDescription 
processInstanceJobDescription) {
+            return processInstanceJobDescription.processInstanceId();
+        } else if (jobDescription instanceof UserTaskInstanceJobDescription 
userTaskInstanceJobDescription) {
+            return userTaskInstanceJobDescription.processInstanceId();
+        }
+        return null;
+    }
+
     public static <T extends JobDescription> String resolveProcessId(T 
jobDescription) {
         String processId = null;
         if (jobDescription instanceof ProcessJobDescription 
processJobDescription) {
diff --git 
a/kogito-jobs/jobs-common-embedded/src/test/java/org/kie/kogito/app/jobs/impl/VertxJobSchedulerLockTest.java
 
b/kogito-jobs/jobs-common-embedded/src/test/java/org/kie/kogito/app/jobs/impl/VertxJobSchedulerLockTest.java
new file mode 100644
index 00000000000..a5ec43b08c2
--- /dev/null
+++ 
b/kogito-jobs/jobs-common-embedded/src/test/java/org/kie/kogito/app/jobs/impl/VertxJobSchedulerLockTest.java
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.kie.kogito.app.jobs.impl;
+
+import java.lang.reflect.Field;
+import java.time.Duration;
+import java.time.ZonedDateTime;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.kie.kogito.app.jobs.api.JobDescriptionMerger;
+import org.kie.kogito.app.jobs.api.JobDetailsEventAdapter;
+import org.kie.kogito.app.jobs.api.JobExecutor;
+import org.kie.kogito.app.jobs.api.JobScheduler;
+import org.kie.kogito.app.jobs.api.JobSchedulerBuilder;
+import org.kie.kogito.app.jobs.api.JobTimeoutInterceptor;
+import 
org.kie.kogito.app.jobs.integrations.ProcessInstanceJobDescriptionJobInstanceEventAdapter;
+import 
org.kie.kogito.app.jobs.integrations.ProcessInstanceJobDescriptionMerger;
+import 
org.kie.kogito.app.jobs.integrations.ProcessJobDescriptionJobInstanceEventAdapter;
+import org.kie.kogito.app.jobs.integrations.ProcessJobDescriptionMerger;
+import 
org.kie.kogito.app.jobs.integrations.UserTaskInstanceJobDescriptionJobInstanceEventAdapter;
+import 
org.kie.kogito.app.jobs.integrations.UserTaskInstanceJobDescriptorMerger;
+import org.kie.kogito.app.jobs.spi.JobContext;
+import org.kie.kogito.app.jobs.spi.JobStore;
+import org.kie.kogito.app.jobs.spi.memory.MemoryJobContextFactory;
+import org.kie.kogito.app.jobs.spi.memory.MemoryJobStore;
+import org.kie.kogito.jobs.ExactExpirationTime;
+import org.kie.kogito.jobs.JobDescription;
+import org.kie.kogito.jobs.descriptors.ProcessInstanceJobDescription;
+import org.kie.kogito.jobs.descriptors.ProcessJobDescription;
+import org.kie.kogito.jobs.descriptors.UserTaskInstanceJobDescription;
+import org.kie.kogito.jobs.service.model.JobDetails;
+import org.kie.kogito.process.impl.lock.ProcessInstanceAtomicLockStrategy;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+
+class VertxJobSchedulerLockTest {
+
+    private static final String SERVICE_URL = "http://localhost:8080";;
+
+    private JobScheduler jobScheduler;
+
+    @AfterEach
+    void closeScheduler() {
+        if (jobScheduler != null) {
+            jobScheduler.close();
+            jobScheduler = null;
+        }
+    }
+
+    /**
+     * Records what the Vert.x worker thread sees when a transaction-like 
interceptor starts.
+     */
+    private static class LockObservation {
+        final AtomicBoolean executed = new AtomicBoolean();
+        final AtomicBoolean lockedKey = new AtomicBoolean();
+        final AtomicBoolean lockedOtherKey = new AtomicBoolean();
+    }
+
+    private JobScheduler newScheduler(JobStore jobStore, 
JobDetailsEventAdapter adapter, JobDescriptionMerger merger,
+            LatchExecutionJobSchedulerListener listener, JobExecutor executor, 
JobTimeoutInterceptor... interceptors) {
+        jobScheduler = JobSchedulerBuilder.newJobSchedulerBuilder()
+                .withJobExecutors(executor)
+                .withJobEventAdapters(adapter)
+                .withEventPublishers(new TestEventPublisher())
+                .withJobContextFactory(new MemoryJobContextFactory())
+                .withJobStore(jobStore)
+                .withJobSchedulerListeners(listener)
+                .withJobDescriptorMergers(merger)
+                .withTimeoutInterceptor(interceptors)
+                .build();
+        jobScheduler.init();
+        return jobScheduler;
+    }
+
+    /**
+     * Schedules the job and observes, from the worker thread and at the point 
where a transaction interceptor would start,
+     * whether the lock for {@code key} and for {@code otherKey} is held. Then 
verifies the lock for {@code key} was released.
+     */
+    private LockObservation runAndObserveLocks(JobDescription jobDescription, 
JobDetailsEventAdapter adapter, JobDescriptionMerger merger,
+            String key, String otherKey) throws Exception {
+        LockObservation observation = new LockObservation();
+        ProcessInstanceAtomicLockStrategy locks = 
ProcessInstanceAtomicLockStrategy.instance();
+
+        JobTimeoutInterceptor observingInterceptor = callable -> () -> {
+            observation.lockedKey.set(locks.isLockedByCurrentThread(key));
+            
observation.lockedOtherKey.set(locks.isLockedByCurrentThread(otherKey));
+            return callable.call();
+        };
+
+        LatchExecutionJobSchedulerListener listener = new 
LatchExecutionJobSchedulerListener();
+        TestJobExecutor executor = new TestJobExecutor();
+        newScheduler(new MemoryJobStore(), adapter, merger, listener, 
executor, observingInterceptor);
+
+        jobScheduler.schedule(jobDescription);
+        listener.waitForExecution(3000L);
+        observation.executed.set(listener.isExecuted() && 
executor.getJobsExecuted().contains(jobDescription.id()));
+
+        // the lock must not leak: it can be acquired from another thread once 
the timeout task finished
+        assertTimeoutPreemptively(Duration.ofSeconds(3), () -> 
locks.executeOperation(key, () -> null));
+        return observation;
+    }
+
+    private static ProcessInstanceJobDescription processInstanceJob(String 
jobId, String processInstanceId, ZonedDateTime when) {
+        return 
ProcessInstanceJobDescription.newProcessInstanceJobDescriptionBuilder()
+                .id(jobId)
+                .timerId("timer-" + jobId)
+                .processId("testProcess")
+                .processInstanceId(processInstanceId)
+                .expirationTime(ExactExpirationTime.of(when))
+                .build();
+    }
+
+    /**
+     * The lock for the process instance must already be held by the worker 
thread when the transaction interceptor starts.
+     */
+    @Test
+    void testProcessInstanceJobLocksBeforeInterceptorRuns() throws Exception {
+        final String jobId = "lock-order-job-1";
+        final String processInstanceId = "pi-lock-order-123";
+
+        LockObservation observation = runAndObserveLocks(
+                processInstanceJob(jobId, processInstanceId, 
ZonedDateTime.now().plus(Duration.ofMillis(50))),
+                new 
ProcessInstanceJobDescriptionJobInstanceEventAdapter(SERVICE_URL),
+                new ProcessInstanceJobDescriptionMerger(),
+                processInstanceId, jobId);
+
+        assertThat(observation.executed).isTrue();
+        assertThat(observation.lockedKey).as("process instance lock must be 
held when the transaction interceptor starts").isTrue();
+        assertThat(observation.lockedOtherKey).as("job id must not be used as 
lock key").isFalse();
+    }
+
+    /**
+     * User task timers must lock on the owning process instance id, not on 
the job id
+     * (which is what the job correlationId holds for this kind of job).
+     */
+    @Test
+    void testUserTaskJobLocksOnProcessInstanceId() throws Exception {
+        final String jobId = "usertask-job-4";
+        final String processInstanceId = "pi-usertask-789";
+
+        UserTaskInstanceJobDescription jobDescription = 
UserTaskInstanceJobDescription.newUserTaskInstanceJobDescriptionBuilder()
+                .id(jobId)
+                .userTaskInstanceId("ut-1")
+                .metadata(new HashMap<>())
+                .processId("testProcess")
+                .processInstanceId(processInstanceId)
+                
.expirationTime(ExactExpirationTime.of(ZonedDateTime.now().plus(Duration.ofMillis(50))))
+                .build();
+
+        LockObservation observation = runAndObserveLocks(jobDescription,
+                new 
UserTaskInstanceJobDescriptionJobInstanceEventAdapter(SERVICE_URL),
+                new UserTaskInstanceJobDescriptorMerger(),
+                processInstanceId, jobId);
+
+        assertThat(observation.executed).isTrue();
+        assertThat(observation.lockedKey).as("process instance lock must be 
held for user task jobs").isTrue();
+        assertThat(observation.lockedOtherKey).as("job id must not be used as 
lock key").isFalse();
+    }
+
+    /**
+     * Process definition timers (e.g. start timers) are not bound to any 
process instance,
+     * so no lock must be taken, in particular not on the process id.
+     */
+    @Test
+    void testProcessDefinitionJobIsNotLocked() throws Exception {
+        final String processId = "testProcessDefinition";
+
+        ProcessJobDescription jobDescription = ProcessJobDescription.of(
+                
ExactExpirationTime.of(ZonedDateTime.now().plus(Duration.ofMillis(50))), 
processId, "1.0");
+
+        LockObservation observation = runAndObserveLocks(jobDescription,
+                new ProcessJobDescriptionJobInstanceEventAdapter(SERVICE_URL),
+                new ProcessJobDescriptionMerger(),
+                processId, jobDescription.id());
+
+        assertThat(observation.executed).isTrue();
+        assertThat(observation.lockedKey).as("process definition jobs must not 
lock on the process id").isFalse();
+        assertThat(observation.lockedOtherKey).as("process definition jobs 
must not lock on the job id").isFalse();
+    }
+
+    /**
+     * Jobs that are not related to processes at all must not take any lock.
+     */
+    @Test
+    void testJobWithoutProcessInstanceIsNotLocked() throws Exception {
+        final String jobId = "no-process-job-3";
+
+        LockObservation observation = runAndObserveLocks(
+                new TestJobDescription(jobId, 
ZonedDateTime.now().plus(Duration.ofMillis(50))),
+                new TestJobDetailsEventAdapter(),
+                new TestJobDescriptionMerger(),
+                jobId, jobId);
+
+        assertThat(observation.executed).isTrue();
+        assertThat(observation.lockedKey).as("jobs without a process instance 
must not be locked").isFalse();
+    }
+
+    /**
+     * While another thread holds the process instance lock (e.g. an incoming 
REST request), the timer thread must wait for it
+     * BEFORE the transaction interceptor starts, so the transaction observes 
the latest committed state rather than a stale snapshot.
+     */
+    @Test
+    void testConcurrentLockContentionGuaranteesFreshTransactionSnapshot() 
throws Exception {
+        final String jobId = "concurrent-lock-job-2";
+        final String processInstanceId = "pi-concurrent-456";
+
+        final AtomicInteger simulatedDbVersion = new AtomicInteger(1);
+        final AtomicInteger versionObservedAtTxStart = new AtomicInteger(-1);
+        final CountDownLatch jobExecuted = new CountDownLatch(1);
+        // the periodic job store sync goes through the interceptors too, so 
the job itself is what tells us which transaction is relevant
+        final ThreadLocal<Integer> versionAtCurrentTxStart = new 
ThreadLocal<>();
+
+        JobTimeoutInterceptor mockTransactionInterceptor = callable -> () -> {
+            // the DB version observed at the exact moment the transaction 
starts
+            versionAtCurrentTxStart.set(simulatedDbVersion.get());
+            try {
+                return callable.call();
+            } finally {
+                versionAtCurrentTxStart.remove();
+            }
+        };
+
+        JobExecutor executor = new JobExecutor() {
+            @Override
+            public boolean accept(JobDetails jobDetails) {
+                return true;
+            }
+
+            @Override
+            public void execute(JobDetails jobDetails) {
+                versionObservedAtTxStart.set(versionAtCurrentTxStart.get());
+                jobExecuted.countDown();
+            }
+        };
+
+        LatchExecutionJobSchedulerListener listener = new 
LatchExecutionJobSchedulerListener();
+        newScheduler(new MemoryJobStore(), new 
ProcessInstanceJobDescriptionJobInstanceEventAdapter(SERVICE_URL),
+                new ProcessInstanceJobDescriptionMerger(), listener, executor, 
mockTransactionInterceptor);
+
+        CountDownLatch externalLockAcquired = new CountDownLatch(1);
+        CountDownLatch externalRelease = new CountDownLatch(1);
+
+        ExecutorService backgroundExecutor = 
Executors.newSingleThreadExecutor();
+        try {
+            Future<?> external = backgroundExecutor.submit(() -> 
ProcessInstanceAtomicLockStrategy.instance().executeOperation(processInstanceId,
 () -> {
+                externalLockAcquired.countDown();
+                try {
+                    externalRelease.await(5, TimeUnit.SECONDS);
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                }
+                // update the state/version before releasing the lock
+                simulatedDbVersion.set(2);
+                return null;
+            }));
+
+            assertThat(externalLockAcquired.await(2, 
TimeUnit.SECONDS)).isTrue();
+
+            // the timer fires while the external thread still holds the lock
+            jobScheduler.schedule(processInstanceJob(jobId, processInstanceId, 
ZonedDateTime.now().plus(Duration.ofMillis(10))));
+
+            // the job (and so its transaction) must not run while the lock is 
held by someone else
+            assertThat(jobExecuted.await(300, TimeUnit.MILLISECONDS))
+                    .as("job must not run while the process instance lock is 
held by another thread")
+                    .isFalse();
+
+            externalRelease.countDown();
+            external.get(3, TimeUnit.SECONDS);
+        } finally {
+            externalRelease.countDown();
+            backgroundExecutor.shutdownNow();
+        }
+
+        assertThat(jobExecuted.await(3, TimeUnit.SECONDS)).isTrue();
+        assertThat(versionObservedAtTxStart.get())
+                .as("transaction must start AFTER the lock is acquired, 
observing the latest state (version 2)")
+                .isEqualTo(2);
+    }
+
+    /**
+     * A timer that finds its job should not run (e.g. it was rescheduled in 
the meantime) must only drop its own entry
+     * from the scheduled jobs, not the entry of the timer that replaced it 
under the same key.
+     */
+    @Test
+    void testShouldNotRunDoesNotRemoveEntryOfRescheduledTimer() throws 
Exception {
+        final String jobId = "reschedule-job-5";
+        final String processInstanceId = "pi-reschedule-321";
+
+        CountDownLatch shouldRunEntered = new CountDownLatch(1);
+        CountDownLatch shouldRunRelease = new CountDownLatch(1);
+        AtomicBoolean firstCall = new AtomicBoolean(true);
+
+        // the first timeout is held inside shouldRun and then told not to 
run, later calls behave normally
+        JobStore blockingStore = new MemoryJobStore() {
+            @Override
+            public boolean shouldRun(JobContext jobContext, String id) {
+                if (firstCall.compareAndSet(true, false)) {
+                    shouldRunEntered.countDown();
+                    try {
+                        shouldRunRelease.await(5, TimeUnit.SECONDS);
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    }
+                    return false;
+                }
+                return super.shouldRun(jobContext, id);
+            }
+        };
+
+        newScheduler(blockingStore, new 
ProcessInstanceJobDescriptionJobInstanceEventAdapter(SERVICE_URL),
+                new ProcessInstanceJobDescriptionMerger(), new 
LatchExecutionJobSchedulerListener(), new TestJobExecutor());
+
+        jobScheduler.schedule(processInstanceJob(jobId, processInstanceId, 
ZonedDateTime.now().plus(Duration.ofMillis(50))));
+        assertThat(shouldRunEntered.await(3, TimeUnit.SECONDS)).isTrue();
+
+        // while the first timer is still in flight, the job is rescheduled 
and gets a new timer under the same key
+        jobScheduler.reschedule(processInstanceJob(jobId, processInstanceId, 
ZonedDateTime.now().plus(Duration.ofMinutes(10))));
+        shouldRunRelease.countDown();
+
+        Map<String, ?> jobsScheduled = jobsScheduled();
+        String mapKey = jobId + "-0";
+        assertThat(jobsScheduled).containsKey(mapKey);
+        // the first timeout has now completed its "should not run" handling, 
the entry of the rescheduled timer must survive
+        await().during(500, TimeUnit.MILLISECONDS).atMost(2, TimeUnit.SECONDS)
+                .untilAsserted(() -> 
assertThat(jobsScheduled).containsKey(mapKey));
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, ?> jobsScheduled() throws Exception {
+        Field field = 
VertxJobScheduler.class.getDeclaredField("jobsScheduled");
+        field.setAccessible(true);
+        return (Map<String, ?>) field.get(jobScheduler);
+    }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to