This is an automated email from the ASF dual-hosted git repository.
DaanHoogland pushed a commit to branch 4.22
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.22 by this push:
new 7d45dcb4301 Migrations triggered by Maintenance (system) not reported
on VM events list (#13931)
7d45dcb4301 is described below
commit 7d45dcb430182c991ea713105ec9c5dc47b42e78
Author: Abhisar Sinha <[email protected]>
AuthorDate: Wed Sep 2 12:36:47 2026 +0530
Migrations triggered by Maintenance (system) not reported on VM events list
(#13931)
Co-authored-by: mprokopchuk <[email protected]>
Co-authored-by: Copilot Autofix powered by AI
<[email protected]>
---
.../main/java/com/cloud/event/dao/EventDao.java | 13 +++
.../java/com/cloud/event/dao/EventDaoImpl.java | 21 +++-
.../java/com/cloud/utils/db/GenericDaoBase.java | 11 ++-
.../java/com/cloud/event/ActionEventUtils.java | 14 +++
.../com/cloud/ha/HighAvailabilityManagerImpl.java | 109 +++++++++++++++++++--
.../cloud/ha/HighAvailabilityManagerImplTest.java | 8 +-
6 files changed, 164 insertions(+), 12 deletions(-)
diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java
b/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java
index c50451b03e4..03716237cdb 100644
--- a/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java
+++ b/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java
@@ -19,6 +19,7 @@ package com.cloud.event.dao;
import java.util.Date;
import java.util.List;
+import com.cloud.event.Event;
import com.cloud.event.EventVO;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericDao;
@@ -31,6 +32,18 @@ public interface EventDao extends GenericDao<EventVO, Long> {
EventVO findCompletedEvent(long startId);
+ /**
+ * Finds the last non-archived start event matching the specified criteria.
+ * Events are ordered by ID in descending order, returning the most recent
one.
+ *
+ * @param type the event type to search for
+ * @param state the event state to search for (e.g., {@link
Event.State#Scheduled})
+ * @param resourceId the resource ID associated with the event
+ * @param resourceType the resource type associated with the event
+ * @return the most recent EventVO matching the criteria, or null if not
found
+ */
+ EventVO findLastEvent(String type, Event.State state, Long resourceId,
String resourceType);
+
public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String
type, Date startDate, Date endDate, List<Long> accountIds);
public void archiveEvents(List<EventVO> events);
diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java
b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java
index e748e98900e..b66da14292e 100644
--- a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java
@@ -19,7 +19,6 @@ package com.cloud.event.dao;
import java.util.Date;
import java.util.List;
-
import org.springframework.stereotype.Component;
import com.cloud.event.Event.State;
@@ -35,6 +34,7 @@ import com.cloud.utils.db.TransactionLegacy;
public class EventDaoImpl extends GenericDaoBase<EventVO, Long> implements
EventDao {
protected final SearchBuilder<EventVO> CompletedEventSearch;
protected final SearchBuilder<EventVO> ToArchiveOrDeleteEventSearch;
+ protected final SearchBuilder<EventVO> LastStartEventSearch;
public EventDaoImpl() {
CompletedEventSearch = createSearchBuilder();
@@ -51,6 +51,14 @@ public class EventDaoImpl extends GenericDaoBase<EventVO,
Long> implements Event
ToArchiveOrDeleteEventSearch.and("createdDateL",
ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LTEQ);
ToArchiveOrDeleteEventSearch.and("archived",
ToArchiveOrDeleteEventSearch.entity().getArchived(), Op.EQ);
ToArchiveOrDeleteEventSearch.done();
+
+ LastStartEventSearch = createSearchBuilder();
+ LastStartEventSearch.and("type",
LastStartEventSearch.entity().getType(), Op.EQ);
+ LastStartEventSearch.and("state",
LastStartEventSearch.entity().getState(), Op.EQ);
+ LastStartEventSearch.and("resourceId",
LastStartEventSearch.entity().getResourceId(), Op.EQ);
+ LastStartEventSearch.and("resourceType",
LastStartEventSearch.entity().getResourceType(), Op.EQ);
+ LastStartEventSearch.and("archived",
LastStartEventSearch.entity().getArchived(), Op.EQ);
+ LastStartEventSearch.done();
}
@Override
@@ -77,6 +85,17 @@ public class EventDaoImpl extends GenericDaoBase<EventVO,
Long> implements Event
return findOneIncludingRemovedBy(sc);
}
+ @Override
+ public EventVO findLastEvent(String type, State state, Long resourceId,
String resourceType) {
+ SearchCriteria<EventVO> sc = LastStartEventSearch.create();
+ sc.setParameters("type", type);
+ sc.setParameters("state", state);
+ sc.setParameters("resourceId", resourceId);
+ sc.setParameters("resourceType", resourceType);
+ sc.setParameters("archived", false);
+ return findLastOneBy(sc);
+ }
+
@Override
public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String
type, Date startDate, Date endDate, List<Long> accountIds) {
SearchCriteria<EventVO> sc = ToArchiveOrDeleteEventSearch.create();
diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java
b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java
index dcd863465d1..dad1877adbc 100644
--- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java
+++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java
@@ -929,7 +929,7 @@ public abstract class GenericDaoBase<T, ID extends
Serializable> extends Compone
protected T findOneIncludingRemovedBy(final SearchCriteria<T> sc) {
Filter filter = new Filter(1, true);
List<T> results = searchIncludingRemoved(sc, filter, null, false);
- assert results.size() <= 1 : "Didn't the limiting worked?";
+ assert results.size() <= 1 : "Didn't the limiting work?";
return results.size() == 0 ? null : results.get(0);
}
@@ -949,6 +949,15 @@ public abstract class GenericDaoBase<T, ID extends
Serializable> extends Compone
return results.isEmpty() ? null : results.get(0);
}
+ @DB()
+ protected T findLastOneBy(SearchCriteria<T> sc) {
+ sc = checkAndSetRemovedIsNull(sc);
+ Filter filter = new Filter(_entityBeanType, "id", Boolean.FALSE, 0L,
1L);
+ List<T> results = searchIncludingRemoved(sc, filter, null, false);
+ assert results.size() <= 1 : "Didn't the limiting work?";
+ return results.size() == 0 ? null : results.get(0);
+ }
+
@DB()
public List<T> listBy(SearchCriteria<T> sc, final Filter filter) {
sc = checkAndSetRemovedIsNull(sc);
diff --git a/server/src/main/java/com/cloud/event/ActionEventUtils.java
b/server/src/main/java/com/cloud/event/ActionEventUtils.java
index ae77446a856..0ebb266fd7c 100644
--- a/server/src/main/java/com/cloud/event/ActionEventUtils.java
+++ b/server/src/main/java/com/cloud/event/ActionEventUtils.java
@@ -400,6 +400,20 @@ public class ActionEventUtils {
return account.getDomainId();
}
+ /**
+ * Retrieves the last non-archived event matching the specified criteria.
+ *
+ * @param type the event type to search for
+ * @param state the event state to search for (e.g., {@link
Event.State#Scheduled})
+ * @param resourceId the resource ID associated with the event
+ * @param resourceType the resource type associated with the event
+ * @return the most recent EventVO matching the criteria, or null if not
found
+ * @see EventDao#findLastEvent(String, Event.State, Long, String)
+ */
+ public static EventVO getLastEvent(String type, Event.State state, Long
resourceId, String resourceType) {
+ return s_eventDao.findLastEvent(type, state, resourceId, resourceType);
+ }
+
private static void populateFirstClassEntities(Map<String, String>
eventDescription){
CallContext context = CallContext.current();
diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java
b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java
index 755de00dec2..66ab4b9ffee 100644
--- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java
+++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java
@@ -17,6 +17,7 @@
package com.cloud.ha;
import static org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone;
+import static com.cloud.event.Event.State;
import java.util.ArrayList;
import java.util.Arrays;
@@ -32,8 +33,14 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
-import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.context.CallContext;
+import com.cloud.event.ActionEventUtils;
+import com.cloud.event.Event;
+import com.cloud.event.EventTypes;
+import com.cloud.event.EventVO;
+import com.cloud.user.Account;
+import com.cloud.user.User;
+import org.apache.cloudstack.api.ApiCommandResourceType;
import
org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
@@ -452,9 +459,16 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
}
Long hostId = VirtualMachine.State.Migrating.equals(vm.getState()) ?
vm.getLastHostId() : vm.getHostId();
- final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(),
WorkType.Migration, Step.Scheduled, vm.getHostId(), vm.getState(), 0,
vm.getUpdated(), reasonType);
+ final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(),
WorkType.Migration, Step.Scheduled, hostId, vm.getState(), 0, vm.getUpdated(),
reasonType);
_haDao.persist(work);
- logger.info("Scheduled migration work of VM {} from host {} with
HAWork {}", vm, _hostDao.findById(vm.getHostId()), work);
+
+ HostVO host = _hostDao.findById(hostId);
+ logger.info(String.format("Scheduled migration work of VM %s from host
%s with HAWork %s", vm, host, work));
+ String hostName =
Optional.ofNullable(host).map(HostVO::getName).orElse("N/A");
+ String msg = String.format("Scheduled migration work of VM %s from
host %s (%s) with HAWork %s (attempt %s of %s)",
+ vm.getHostName(), hostId, hostName, work.getId(),
work.getTimesTried() + 1, _maxRetries);
+ createEvent(vm.getId(), ApiCommandResourceType.VirtualMachine,
EventTypes.EVENT_VM_MIGRATE, msg,
+ State.Scheduled, EventVO.LEVEL_INFO);
wakeupWorkers();
return true;
}
@@ -862,18 +876,77 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
return true;
}
+ /**
+ * Creates an event for {@link ApiCommandResourceType} operations.
+ * This is a fail-safe helper method for logging purposes - exceptions are
caught and logged.
+ *
+ * @param resourceId the resource ID
+ * @param resourceType the event resource type ({@link
ApiCommandResourceType})
+ * @param type the event type ({@link EventTypes})
+ * @param description the event description
+ * @param state the event state ({@link Event.State})
+ * @param level the event level (e.g., {@link EventVO#LEVEL_INFO}
or {@link EventVO#LEVEL_ERROR})
+ */
+ private void createEvent(Long resourceId, ApiCommandResourceType
resourceType, String type, String description,
+ State state, String level) {
+ try {
+ String resourceTypeStr = resourceType.toString();
+ Long userId = User.UID_SYSTEM;
+ Long accountId = Account.ACCOUNT_ID_SYSTEM;
+ if (ApiCommandResourceType.VirtualMachine.equals(resourceType) &&
resourceId != null) {
+ VMInstanceVO vm = _instanceDao.findById(resourceId);
+ if (vm != null) {
+ accountId = vm.getAccountId();
+ }
+ }
+ long startEventId = state == State.Scheduled ? 0L
+ : Optional.ofNullable(ActionEventUtils.getLastEvent(type,
State.Scheduled, resourceId,
+ resourceTypeStr))
+ .map(EventVO::getId).orElse(0L);
+
+ switch (state) {
+ case Started:
+ ActionEventUtils.onStartedActionEvent(userId, accountId,
type, description, resourceId,
+ resourceTypeStr, true, startEventId);
+ break;
+ case Scheduled:
+ ActionEventUtils.onScheduledActionEvent(userId, accountId,
type, description, resourceId,
+ resourceTypeStr, true, startEventId);
+ break;
+ case Completed:
+ ActionEventUtils.onCompletedActionEvent(userId, accountId,
level, type, true,
+ description, resourceId, resourceTypeStr,
startEventId);
+ break;
+ default:
+ throw new CloudRuntimeException("Unsupported event state:
" + state);
+ }
+ } catch (Exception e) {
+ logger.error(String.format("Failed to create event for VM: %s,
command: %s, state: %s, level: %s",
+ resourceId, type, state, level), e);
+ }
+ }
+
public Long migrate(final HaWorkVO work) {
logger.debug("MIGRATE with HA WORK");
long vmId = work.getInstanceId();
long srcHostId = work.getHostId();
HostVO srcHost = _hostDao.findById(srcHostId);
+ ApiCommandResourceType resourceType =
ApiCommandResourceType.VirtualMachine;
+ String eventType = EventTypes.EVENT_VM_MIGRATE;
+ int attemptNumber = work.getTimesTried() + 1;
VMInstanceVO vm = _instanceDao.findById(vmId);
if (vm == null) {
- logger.info("Unable to find vm: {}, skipping migrate.", vmId);
+ String msg = String.format("Unable to find vm %s, skipping
migration. HA Work %s (attempt %s of %s)",
+ vmId, work.getId(), attemptNumber, _maxRetries);
+ logger.info(msg);
+ createEvent(vmId, resourceType, eventType, msg, State.Completed,
EventVO.LEVEL_ERROR);
return null;
}
if (checkAndCancelWorkIfNeeded(work)) {
+ String msg = String.format("Cancelled migration for vm %s as it is
not needed anymore. HA Work %s (attempt %s of %s)",
+ vm.getHostName(), work.getId(), attemptNumber,
_maxRetries);
+ createEvent(vmId, resourceType, eventType, msg, State.Completed,
EventVO.LEVEL_ERROR);
return null;
}
logger.info("Migration attempt: for {} from {}. Starting attempt:
{}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries);
@@ -883,23 +956,41 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
return null;
}
if (VirtualMachine.State.Running.equals(vm.getState()) && srcHostId !=
vm.getHostId()) {
- logger.info("VM {} is running on a different host {}, skipping
migration", vm, vm.getHostId());
+ String vmHostName =
Optional.ofNullable(_hostDao.findById(vm.getHostId())).map(HostVO::getName)
+ .orElse("N/A");
+ String msg = String.format("VM %s is running on a different host
(%s), skipping migration. HA Work %s (attempt %s of %s)",
+ vm.getHostName(), vmHostName, work.getId(), attemptNumber,
_maxRetries);
+ logger.info(msg);
+ createEvent(vmId, resourceType, eventType, msg, State.Completed,
EventVO.LEVEL_ERROR);
return null;
}
-
+ logger.info(String.format("Migration attempt: for VM %s from host %s.
Starting attempt: %d/%d times.",
+ vm, srcHost, attemptNumber, _maxRetries));
try {
+ String vmHostName =
Optional.ofNullable(_hostDao.findById(vm.getHostId())).map(HostVO::getName)
+ .orElse("N/A");
+ String msg = String.format("Starting migration from host %s. HA
Work %s (attempt %s of %s)",
+ vmHostName, work.getId(), attemptNumber, _maxRetries);
+ createEvent(vmId, resourceType, eventType, msg, State.Started,
EventVO.LEVEL_INFO);
work.setStep(Step.Migrating);
_haDao.update(work.getId(), work);
-
// First try starting the vm with its original planner, if it
doesn't succeed send HAPlanner as its an emergency.
_itMgr.migrateAway(vm.getUuid(), srcHostId);
+ msg = String.format("Completed migration. HA Work %s (attempt %s
of %s)", work.getId(), attemptNumber, _maxRetries);
+ createEvent(vmId, resourceType, eventType, msg, State.Completed,
EventVO.LEVEL_INFO);
return null;
} catch (InsufficientServerCapacityException e) {
- logger.warn("Migration attempt: Insufficient capacity for
migrating a VM {} from source host {}. Exception: {}", vm, srcHost,
e.getMessage());
+ String msg = String.format("Migration attempt: Insufficient
capacity for migrating a VM %s from source host %s. HA Work %s (attempt %s of
%s)",
+ vm.getHostName(), srcHost, work.getId(), attemptNumber,
_maxRetries);
+ logger.warn(msg);
_resourceMgr.migrateAwayFailed(srcHostId, vmId);
+ createEvent(vmId, resourceType, eventType, msg, State.Completed,
EventVO.LEVEL_ERROR);
return (System.currentTimeMillis() >> 10) + _migrateRetryInterval;
} catch (Exception e) {
- logger.warn("Migration attempt: Unexpected exception occurred when
attempting migration of {} {}", vm, e.getMessage());
+ String msg = String.format("Migration attempt: Unexpected
exception occurred when attempting migration of vm %s. HA Work %s (attempt %s
of %s)",
+ vm.getHostName(), work.getId(), attemptNumber,
_maxRetries);
+ logger.warn(msg);
+ createEvent(vmId, resourceType, eventType, msg, State.Completed,
EventVO.LEVEL_ERROR);
throw e;
}
}
diff --git
a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java
b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java
index 626f2cda172..1fe4263afc5 100644
--- a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java
+++ b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java
@@ -78,6 +78,7 @@ import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineManager;
import com.cloud.vm.dao.VMInstanceDao;
+import org.springframework.test.util.ReflectionTestUtils;
@RunWith(MockitoJUnitRunner.class)
public class HighAvailabilityManagerImplTest {
@@ -309,7 +310,12 @@ public class HighAvailabilityManagerImplTest {
Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User);
Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running);
Mockito.when(vm.getHostId()).thenReturn(1L);
-
Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));
+
+ Mockito.when(_haDao.persist((HaWorkVO)
Mockito.any())).thenAnswer(invocation -> {
+ HaWorkVO haWork = invocation.getArgument(0);
+ ReflectionTestUtils.setField(haWork, "id", 1L);
+ return haWork;
+ });
ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
highAvailabilityManager.VmHaEnabled = haEnabled;
