This is an automated email from the ASF dual-hosted git repository.
rohit pushed a commit to branch 4.20
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.20 by this push:
new 330ed25a6c1 Support to enable/disable VM High Availability manager and
related alerts (#10118)
330ed25a6c1 is described below
commit 330ed25a6c1ba357bf361f8c6021d4db57d62a7e
Author: Suresh Kumar Anaparti <sureshkumar.anapa...@gmail.com>
AuthorDate: Thu Dec 26 17:45:32 2024 +0530
Support to enable/disable VM High Availability manager and related alerts
(#10118)
- Adds new config 'vm.ha.enabled' with Zone scope, to enable/disable VM
High Availability manager. This is enable by default (for backward
compatibilty).
When enabled, the VM HA WorkItems (for VM Stop, Restart, Migration,
Destroy) can be created and the scheduled items are executed.
When disabled, new VM HA WorkItems are not allowed and the scheduled
items are retried until max retries configured at 'vm.ha.migration.max.retries'
(executed in case HA is re-enabled during retry attempts), and then purged
after 'time.between.failures' by the cleanup thread that runs regularly at
'time.between.cleanup'.
- Adds new config 'vm.ha.alerts.enabled' with Zone scope, to enable/disable
alerts for the VM HA operations. This is enabled by default.
---
.../java/com/cloud/ha/HighAvailabilityManager.java | 6 +-
.../com/cloud/ha/HighAvailabilityManagerImpl.java | 129 +++++++++++++++--
.../com/cloud/resource/ResourceManagerImpl.java | 10 +-
.../cloud/ha/HighAvailabilityManagerImplTest.java | 156 +++++++++++++++++++++
4 files changed, 281 insertions(+), 20 deletions(-)
diff --git
a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java
b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java
index ae47b1d76ed..728f5a2b180 100644
---
a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java
+++
b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java
@@ -32,7 +32,7 @@ import java.util.List;
*/
public interface HighAvailabilityManager extends Manager {
- public ConfigKey<Boolean> ForceHA = new ConfigKey<>("Advanced",
Boolean.class, "force.ha", "false",
+ ConfigKey<Boolean> ForceHA = new ConfigKey<>("Advanced", Boolean.class,
"force.ha", "false",
"Force High-Availability to happen even if the VM says no.", true,
Cluster);
ConfigKey<Integer> HAWorkers = new ConfigKey<>("Advanced", Integer.class,
"ha.workers", "5",
@@ -112,7 +112,7 @@ public interface HighAvailabilityManager extends Manager {
void cancelDestroy(VMInstanceVO vm, Long hostId);
- void scheduleDestroy(VMInstanceVO vm, long hostId);
+ boolean scheduleDestroy(VMInstanceVO vm, long hostId);
/**
* Schedule restarts for all vms running on the host.
@@ -143,7 +143,7 @@ public interface HighAvailabilityManager extends Manager {
* @param host host the virtual machine is on.
* @param type which type of stop is requested.
*/
- void scheduleStop(VMInstanceVO vm, long hostId, WorkType type);
+ boolean scheduleStop(VMInstanceVO vm, long hostId, WorkType type);
void cancelScheduledMigrations(HostVO host);
diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java
b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java
index d8fc99a0934..20435f48b52 100644
--- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java
+++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java
@@ -16,6 +16,8 @@
// under the License.
package com.cloud.ha;
+import static org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone;
+
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -121,6 +123,16 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
"Total number of attempts for trying migration of a VM.",
true, ConfigKey.Scope.Global);
+ public static ConfigKey<Boolean> VmHaEnabled = new ConfigKey<>("Advanced",
Boolean.class, "vm.ha.enabled", "true",
+ "Enable/Disable VM High Availability manager, it is enabled by
default."
+ + " When enabled, the VM HA WorkItems (for VM Stop,
Restart, Migration, Destroy) can be created and the scheduled items are
executed; and"
+ + " When disabled, new VM HA WorkItems are not allowed and
the scheduled items are retried until max retries configured at
'vm.ha.migration.max.retries'"
+ + " (executed in case HA is re-enabled during retry
attempts), and then purged after 'time.between.failures' by the cleanup thread
that runs"
+ + " regularly at 'time.between.cleanup'", true, Zone);
+
+ protected static ConfigKey<Boolean> VmHaAlertsEnabled = new
ConfigKey<>("Advanced", Boolean.class, "vm.ha.alerts.enabled", "true",
+ "Enable/Disable alerts for the VM HA operations, it is enabled by
default.", true, Zone);
+
WorkerThread[] _workers;
boolean _stopped;
long _timeToSleep;
@@ -185,7 +197,6 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
_haPlanners = haPlanners;
}
-
@Inject
AgentManager _agentMgr;
@Inject
@@ -231,6 +242,15 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
return Status.Alert;
}
+ if (!VmHaEnabled.valueIn(host.getDataCenterId())) {
+ String message = String.format("Unable to investigate the host %s
(%d), VM high availability manager is disabled.", host.getName(), hostId);
+ if (logger.isDebugEnabled()) {
+ logger.debug(message);
+ }
+ sendHostAlert(host, message);
+ return Status.Alert;
+ }
+
Status hostState = null;
for (Investigator investigator : investigators) {
hostState = investigator.isAgentAlive(host);
@@ -260,6 +280,15 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
return;
}
+ if (!VmHaEnabled.valueIn(host.getDataCenterId())) {
+ String message = String.format("Unable to schedule restart for VMs
on host %s (%d), VM high availability manager is disabled.", host.getName(),
host.getId());
+ if (logger.isDebugEnabled()) {
+ logger.debug(message);
+ }
+ sendHostAlert(host, message);
+ return;
+ }
+
logger.warn("Scheduling restart for VMs on host " + host.getId() + "-"
+ host.getName());
final List<VMInstanceVO> vms = _instanceDao.listByHostId(host.getId());
@@ -314,12 +343,21 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
}
@Override
- public void scheduleStop(VMInstanceVO vm, long hostId, WorkType type) {
+ public boolean scheduleStop(VMInstanceVO vm, long hostId, WorkType type) {
assert (type == WorkType.CheckStop || type == WorkType.ForceStop ||
type == WorkType.Stop);
if (_haDao.hasBeenScheduled(vm.getId(), type)) {
logger.info("There's already a job scheduled to stop " + vm);
- return;
+ return false;
+ }
+
+ if (!VmHaEnabled.valueIn(vm.getDataCenterId())) {
+ String message = String.format("Unable to schedule stop for the VM
%s (%d) on host %d, VM high availability manager is disabled.", vm.getName(),
vm.getId(), hostId);
+ if (logger.isDebugEnabled()) {
+ logger.debug(message);
+ }
+ sendVMAlert(vm, message);
+ return false;
}
HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), type,
Step.Scheduled, hostId, vm.getState(), 0, vm.getUpdated());
@@ -328,6 +366,7 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
logger.debug("Scheduled " + work);
}
wakeupWorkers();
+ return true;
}
protected void wakeupWorkers() {
@@ -339,17 +378,37 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
@Override
public boolean scheduleMigration(final VMInstanceVO vm) {
- if (vm.getHostId() != null) {
- final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(),
WorkType.Migration, Step.Scheduled, vm.getHostId(), vm.getState(), 0,
vm.getUpdated());
- _haDao.persist(work);
- logger.info("Scheduled migration work of VM " + vm.getUuid() + "
from host " + _hostDao.findById(vm.getHostId()) + " with HAWork " + work);
- wakeupWorkers();
+ if (vm.getHostId() == null) {
+ return false;
+ }
+
+ if (!VmHaEnabled.valueIn(vm.getDataCenterId())) {
+ String message = String.format("Unable to schedule migration for
the VM %s (%d) on host %d, VM high availability manager is disabled.",
vm.getName(), vm.getId(), vm.getHostId());
+ if (logger.isDebugEnabled()) {
+ logger.debug(message);
+ }
+ sendVMAlert(vm, message);
+ return false;
}
+
+ final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(),
WorkType.Migration, Step.Scheduled, vm.getHostId(), vm.getState(), 0,
vm.getUpdated());
+ _haDao.persist(work);
+ logger.info("Scheduled migration work of VM " + vm.getUuid() + " from
host " + _hostDao.findById(vm.getHostId()) + " with HAWork " + work);
+ wakeupWorkers();
return true;
}
@Override
public void scheduleRestart(VMInstanceVO vm, boolean investigate) {
+ if (!VmHaEnabled.valueIn(vm.getDataCenterId())) {
+ String message = String.format("Unable to schedule restart for the
VM %s (%d), VM high availability manager is disabled.", vm.getName(),
vm.getId());
+ if (logger.isDebugEnabled()) {
+ logger.debug(message);
+ }
+ sendVMAlert(vm, message);
+ return;
+ }
+
logger.debug("HA schedule restart");
Long hostId = vm.getHostId();
if (hostId == null) {
@@ -440,7 +499,6 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
}
wakeupWorkers();
-
}
private void startVm(VirtualMachine vm, Map<VirtualMachineProfile.Param,
Object> params,
@@ -737,13 +795,23 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
}
@Override
- public void scheduleDestroy(VMInstanceVO vm, long hostId) {
+ public boolean scheduleDestroy(VMInstanceVO vm, long hostId) {
+ if (!VmHaEnabled.valueIn(vm.getDataCenterId())) {
+ String message = String.format("Unable to schedule destroy for the
VM %s (%d) on host %d, VM high availability manager is disabled.",
vm.getName(), vm.getId(), hostId);
+ if (logger.isDebugEnabled()) {
+ logger.debug(message);
+ }
+ sendVMAlert(vm, message);
+ return false;
+ }
+
final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(),
WorkType.Destroy, Step.Scheduled, hostId, vm.getState(), 0, vm.getUpdated());
_haDao.persist(work);
if (logger.isDebugEnabled()) {
logger.debug("Scheduled " + work.toString());
}
wakeupWorkers();
+ return true;
}
@Override
@@ -892,7 +960,17 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
private void processWork(final HaWorkVO work) {
final WorkType wt = work.getWorkType();
+ final VMInstanceVO vm = _instanceDao.findById(work.getInstanceId());
try {
+ if (vm != null && !VmHaEnabled.valueIn(vm.getDataCenterId())) {
+ if (logger.isDebugEnabled()) {
+ logger.debug(String.format("VM high availability manager
is disabled, rescheduling the HA work %s, for the VM %s (id) to retry later in
case VM high availability manager is enabled on retry attempt", work,
vm.getName(), vm.getId()));
+ }
+ long nextTime = getRescheduleTime(wt);
+ rescheduleWork(work, nextTime);
+ return;
+ }
+
Long nextTime = null;
if (wt == WorkType.Migration) {
nextTime = migrate(work);
@@ -921,9 +999,10 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
// if restart failed in the middle due to exception, VM state may
has been changed
// recapture into the HA worker so that it can really continue in
it next turn
- VMInstanceVO vm = _instanceDao.findById(work.getInstanceId());
- work.setUpdateTime(vm.getUpdated());
- work.setPreviousState(vm.getState());
+ if (vm != null) {
+ work.setUpdateTime(vm.getUpdated());
+ work.setPreviousState(vm.getState());
+ }
} finally {
if (!Step.Done.equals(work.getStep())) {
if (work.getTimesTried() >= _maxRetries) {
@@ -1128,11 +1207,33 @@ public class HighAvailabilityManagerImpl extends
ManagerBase implements Configur
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey[] {TimeBetweenCleanup, MigrationMaxRetries,
TimeToSleep, TimeBetweenFailures,
StopRetryInterval, RestartRetryInterval, MigrateRetryInterval,
InvestigateRetryInterval,
- HAWorkers, ForceHA, KvmHAFenceHostIfHeartbeatFailsOnStorage};
+ HAWorkers, ForceHA, VmHaEnabled, VmHaAlertsEnabled,
KvmHAFenceHostIfHeartbeatFailsOnStorage};
}
@Override
public int expungeWorkItemsByVmList(List<Long> vmIds, Long batchSize) {
return _haDao.expungeByVmList(vmIds, batchSize);
}
+
+ private void sendVMAlert(VMInstanceVO vm, String message) {
+ if (vm == null || !VmHaAlertsEnabled.valueIn(vm.getDataCenterId())) {
+ return;
+ }
+ AlertManager.AlertType alertType =
AlertManager.AlertType.ALERT_TYPE_USERVM;
+ if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
+ alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER;
+ } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
+ alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY;
+ } else if
(VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) {
+ alertType = AlertManager.AlertType.ALERT_TYPE_SSVM;
+ }
+ _alertMgr.sendAlert(alertType, vm.getDataCenterId(),
vm.getPodIdToDeployIn(), message, message);
+ }
+
+ private void sendHostAlert(HostVO host, String message) {
+ if (host == null ||
!VmHaAlertsEnabled.valueIn(host.getDataCenterId())) {
+ return;
+ }
+ _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST,
host.getDataCenterId(), host.getPodId(), message, message);
+ }
}
diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java
b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java
index 22837389620..94dc9169823 100755
--- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java
+++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java
@@ -41,6 +41,7 @@ import com.cloud.alert.AlertManager;
import com.cloud.cpu.CPU;
import com.cloud.exception.StorageConflictException;
import com.cloud.exception.StorageUnavailableException;
+import com.cloud.ha.HighAvailabilityManagerImpl;
import com.cloud.host.HostTagVO;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeVO;
@@ -1363,6 +1364,11 @@ public class ResourceManagerImpl extends ManagerBase
implements ResourceManager,
throw new CloudRuntimeException("Cannot perform maintain when
resource state is " + hostState + ", hostId = " + hostId);
}
+ final List<VMInstanceVO> vms = _vmDao.listByHostId(hostId);
+ if (CollectionUtils.isNotEmpty(vms) &&
!HighAvailabilityManagerImpl.VmHaEnabled.valueIn(host.getDataCenterId())) {
+ throw new CloudRuntimeException(String.format("Cannot perform
maintain for the host %s (%d) as there are running VMs on it and VM high
availability manager is disabled", host.getName(), hostId));
+ }
+
final MaintainAnswer answer =
(MaintainAnswer)_agentMgr.easySend(hostId, new MaintainCommand());
if (answer == null || !answer.getResult()) {
logger.warn("Unable to send MaintainCommand to host: " + hostId);
@@ -1382,8 +1388,6 @@ public class ResourceManagerImpl extends ManagerBase
implements ResourceManager,
/* TODO: move below to listener */
if (host.getType() == Host.Type.Routing) {
-
- final List<VMInstanceVO> vms = _vmDao.listByHostId(hostId);
if (vms.size() == 0) {
return true;
}
@@ -2841,7 +2845,7 @@ public class ResourceManagerImpl extends ManagerBase
implements ResourceManager,
logger.debug("Cannot transmit host " + host.getId() +
" to Disabled state", e);
}
for (final VMInstanceVO vm : vms) {
- if ((! HighAvailabilityManager.ForceHA.value() &&
!vm.isHaEnabled()) || vm.getState() == State.Stopping) {
+ if ((!HighAvailabilityManager.ForceHA.value() &&
!vm.isHaEnabled()) || vm.getState() == State.Stopping) {
logger.debug(String.format("Stopping %s as a part
of hostDelete for %s",vm, host));
try {
_haMgr.scheduleStop(vm, host.getId(),
WorkType.Stop);
diff --git
a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java
b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java
index 74897967a2f..27b6e1b1b4c 100644
--- a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java
+++ b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java
@@ -16,6 +16,7 @@
// under the License.
package com.cloud.ha;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -32,6 +33,7 @@ import javax.inject.Inject;
import
org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import
org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
+import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.managed.context.ManagedContext;
import org.apache.logging.log4j.LogManager;
@@ -174,10 +176,15 @@ public class HighAvailabilityManagerImplTest {
public void scheduleRestartForVmsOnHost() {
Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing);
Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.KVM);
+ Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);
Mockito.lenient().when(_instanceDao.listByHostId(42l)).thenReturn(Arrays.asList(Mockito.mock(VMInstanceVO.class)));
Mockito.when(_podDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(HostPodVO.class));
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(DataCenterVO.class));
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);
+
highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true);
}
@@ -189,11 +196,25 @@ public class HighAvailabilityManagerImplTest {
highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true);
}
+ @Test
+ public void scheduleRestartForVmsOnHostHADisabled() {
+ Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing);
+
Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.KVM);
+ Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);
+
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);
+
+ highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true);
+ }
+
@Test
public void scheduleRestartForVmsOnHostNonEmptyVMList() {
Mockito.when(hostVO.getId()).thenReturn(1l);
Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing);
Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.XenServer);
+ Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);
List<VMInstanceVO> vms = new ArrayList<VMInstanceVO>();
VMInstanceVO vm1 = Mockito.mock(VMInstanceVO.class);
Mockito.lenient().when(vm1.getHostId()).thenReturn(1l);
@@ -206,6 +227,7 @@ public class HighAvailabilityManagerImplTest {
//Mockito.when(vm2.getInstanceName()).thenReturn("r-2-VM");
Mockito.when(vm2.getType()).thenReturn(VirtualMachine.Type.DomainRouter);
Mockito.when(vm2.isHaEnabled()).thenReturn(true);
+ Mockito.when(vm2.getDataCenterId()).thenReturn(1L);
vms.add(vm2);
Mockito.when(_instanceDao.listByHostId(Mockito.anyLong())).thenReturn(vms);
Mockito.when(_instanceDao.findByUuid(vm1.getUuid())).thenReturn(vm1);
@@ -216,12 +238,125 @@ public class HighAvailabilityManagerImplTest {
Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));
Mockito.when(_serviceOfferingDao.findById(vm1.getServiceOfferingId())).thenReturn(Mockito.mock(ServiceOfferingVO.class));
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);
+
highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true);
}
+ @Test
+ public void scheduleRestartHADisabled() {
+ VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
+ Mockito.when(vm.getId()).thenReturn(1L);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);
+
+ highAvailabilityManager.scheduleRestart(vm, true);
+ }
+
+ @Test
+ public void scheduleRestartHostNotSupported() {
+ VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+ Mockito.when(vm.getHostId()).thenReturn(1L);
+ Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.VMware);
+
+ highAvailabilityManager.scheduleRestart(vm, true);
+ }
+
+ @Test
+ public void scheduleStop() {
+ VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
+ Mockito.when(vm.getId()).thenReturn(1L);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+ Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User);
+ Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running);
+ Mockito.when(_haDao.hasBeenScheduled(vm.getId(),
WorkType.Stop)).thenReturn(false);
+
Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));
+
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);
+
+ assertTrue(highAvailabilityManager.scheduleStop(vm, 1L,
WorkType.Stop));
+ }
+
+ @Test
+ public void scheduleStopHADisabled() {
+ VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
+ Mockito.when(vm.getId()).thenReturn(1L);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+ Mockito.when(_haDao.hasBeenScheduled(vm.getId(),
WorkType.Stop)).thenReturn(false);
+
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);
+
+ assertFalse(highAvailabilityManager.scheduleStop(vm, 1L,
WorkType.Stop));
+ }
+
+ @Test
+ public void scheduleMigration() {
+ VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
+ Mockito.when(vm.getId()).thenReturn(1L);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+ Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User);
+ Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running);
+ Mockito.when(vm.getHostId()).thenReturn(1L);
+
Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));
+
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);
+
+ assertTrue(highAvailabilityManager.scheduleMigration(vm));
+ }
+
+ @Test
+ public void scheduleMigrationHADisabled() {
+ VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
+ Mockito.when(vm.getHostId()).thenReturn(1L);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);
+
+ assertFalse(highAvailabilityManager.scheduleMigration(vm));
+ }
+
+ @Test
+ public void scheduleDestroy() {
+ VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
+ Mockito.when(vm.getId()).thenReturn(1L);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+ Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User);
+ Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running);
+
Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));
+
+ assertTrue(highAvailabilityManager.scheduleDestroy(vm, 1L));
+ }
+
+ @Test
+ public void scheduleDestroyHADisabled() {
+ VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
+ Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);
+
+ assertFalse(highAvailabilityManager.scheduleDestroy(vm, 1L));
+ }
+
@Test
public void investigateHostStatusSuccess() {
Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO);
+ Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);
// Set the list of investigators, CheckOnAgentInvestigator suffices
for now
Investigator investigator =
Mockito.mock(CheckOnAgentInvestigator.class);
List<Investigator> investigators = new ArrayList<Investigator>();
@@ -230,12 +365,17 @@ public class HighAvailabilityManagerImplTest {
// Mock isAgentAlive to return host status as Down
Mockito.when(investigator.isAgentAlive(hostVO)).thenReturn(Status.Down);
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);
+
assertTrue(highAvailabilityManager.investigate(1l) == Status.Down);
}
@Test
public void investigateHostStatusFailure() {
Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO);
+ Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);
// Set the list of investigators, CheckOnAgentInvestigator suffices
for now
// Also no need to mock isAgentAlive() as actual implementation
returns null
Investigator investigator =
Mockito.mock(CheckOnAgentInvestigator.class);
@@ -243,9 +383,25 @@ public class HighAvailabilityManagerImplTest {
investigators.add(investigator);
highAvailabilityManager.setInvestigators(investigators);
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);
+
assertNull(highAvailabilityManager.investigate(1l));
}
+ @Test
+ public void investigateHostStatusHADisabled() {
+ Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO);
+ Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);
+
+ ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
+ highAvailabilityManager.VmHaEnabled = haEnabled;
+
Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);
+
+ assertTrue(highAvailabilityManager.investigate(1L) == Status.Alert);
+ }
+
private void processWorkWithRetryCount(int count, Step expectedStep) {
assertNotNull(processWorkMethod);
HaWorkVO work = new HaWorkVO(1l, VirtualMachine.Type.User,
WorkType.Migration, Step.Scheduled, 1l, VirtualMachine.State.Running, count,
12345678l);
