Arik Hadas has uploaded a new change for review. Change subject: core: add job that runs HA VMs which failed to run ......................................................................
core: add job that runs HA VMs which failed to run This patch adds quartz job which is responsible to run highly available VMs which are down. As a first step it would run HA VMs that could not run because they were locked on last attempt. This patch solves the following problem: when trying to run highly available VM that went down during live storage migration, the RunVmCommand fails because the VM is locked during the migration and the VM won't be started again automatically. Now, when the lock could not be acquired, the newly added job will periodically try to run the VM and when the live storage migration will end, the VM will start. Change-Id: I3d563d05efc6dae40f0de8e7f4b7e47aa84bd787 Bug-Url: https://bugzilla.redhat.com/891634 Signed-off-by: Arik Hadas <[email protected]> --- A backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AutoStartVmsRunner.java M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/Backend.java M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/RunVmCommand.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java M packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql 5 files changed, 80 insertions(+), 0 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/15/18815/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AutoStartVmsRunner.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AutoStartVmsRunner.java new file mode 100644 index 0000000..925f677 --- /dev/null +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/AutoStartVmsRunner.java @@ -0,0 +1,57 @@ +package org.ovirt.engine.core.bll; + +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.ovirt.engine.core.bll.job.ExecutionHandler; +import org.ovirt.engine.core.common.AuditLogType; +import org.ovirt.engine.core.common.action.RunVmParams; +import org.ovirt.engine.core.common.action.VdcActionType; +import org.ovirt.engine.core.common.action.VdcReturnValueBase; +import org.ovirt.engine.core.common.utils.Pair; +import org.ovirt.engine.core.compat.DateTime; +import org.ovirt.engine.core.compat.Guid; +import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector; +import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableBase; +import org.ovirt.engine.core.utils.log.Log; +import org.ovirt.engine.core.utils.log.LogFactory; +import org.ovirt.engine.core.utils.timer.OnTimerMethodAnnotation; + +public class AutoStartVmsRunner { + + private static Log log = LogFactory.getLog(AutoStartVmsRunner.class); + private static AutoStartVmsRunner instance = new AutoStartVmsRunner(); + private ConcurrentLinkedQueue<Pair<DateTime, Guid>> autoStartVmsToRun = new ConcurrentLinkedQueue<>(); + + public static AutoStartVmsRunner getInstance() { + return instance; + } + + private AutoStartVmsRunner() { + } + + @OnTimerMethodAnnotation("startFailedAutoStartVms") + public void startFailedAutoStartVms() { + DateTime now = DateTime.getNow(); + while (autoStartVmsToRun.peek() != null && now.compareTo(autoStartVmsToRun.peek().getFirst()) > 0) { + runVm(autoStartVmsToRun.poll().getSecond()); + } + } + + public void addVmToRun(Guid vmId) { + autoStartVmsToRun.add(new Pair<>(DateTime.getNow(), vmId)); + } + + private void runVm(Guid vmId) { + final VdcReturnValueBase result = Backend.getInstance().runInternalAction(VdcActionType.RunVm, + new RunVmParams(vmId), + ExecutionHandler.createInternalJobContext()); + + // Alert if the restart fails: + if (!result.getSucceeded()) { + final AuditLogableBase event = new AuditLogableBase(); + event.setVmId(vmId); + AuditLogDirector.log(event, AuditLogType.HA_VM_RESTART_FAILED); + // should insert to autoStartVmsToRun again? + } + } +} diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/Backend.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/Backend.java index 7daf7d0..ef6d914 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/Backend.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/Backend.java @@ -98,6 +98,7 @@ private DateTime _startedAt; private static boolean firstInitialization = true; private String poolMonitoringJobId; + private String autoStartVmsRunnerJobId; public static BackendInternal getInstance() { return EjbUtils.findBean(BeanType.BACKEND, BeanProxyType.LOCAL); @@ -236,6 +237,13 @@ vmPoolMonitorIntervalInMinutes, vmPoolMonitorIntervalInMinutes, TimeUnit.MINUTES); + int autoStartVmsRunnerIntervalInSeconds = Config.<Integer> GetValue(ConfigValues.AutoStartVmsRunnerIntervalInSeconds); + autoStartVmsRunnerJobId = + SchedulerUtilQuartzImpl.getInstance().scheduleAFixedDelayJob(AutoStartVmsRunner.getInstance(), + "startFailedAutoStartVms", new Class[] {}, new Object[] {}, + autoStartVmsRunnerIntervalInSeconds, + autoStartVmsRunnerIntervalInSeconds, TimeUnit.SECONDS); + int quotaCacheIntervalInMinutes = Config.<Integer> GetValue(ConfigValues.QuotaCacheIntervalInMinutes); SchedulerUtilQuartzImpl.getInstance().scheduleAFixedDelayJob(QuotaManager.getInstance(), "updateQuotaCache", new Class[] {}, new Object[] {}, diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/RunVmCommand.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/RunVmCommand.java index 293be6d..7eaf08a 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/RunVmCommand.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/RunVmCommand.java @@ -116,6 +116,15 @@ } @Override + protected boolean acquireLockInternal() { + final boolean acquireLock = super.acquireLockInternal(); + if (!acquireLock && getVm().isAutoStartup()) { + AutoStartVmsRunner.getInstance().addVmToRun(getParameters().getVmId()); + } + return acquireLock; + } + + @Override protected VDS getDestinationVds() { if (_destinationVds == null) { Guid vdsId = diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java index c39be8d..8821654 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java @@ -1501,6 +1501,10 @@ @DefaultValueAttribute("true") CloudInitSupported(537), + @TypeConverterAttribute(Integer.class) + @DefaultValueAttribute("60") + AutoStartVmsRunnerIntervalInSeconds(538), + Invalid(65535); private int intValue; diff --git a/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql b/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql index e3d7ffa..78f5cc7 100644 --- a/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql +++ b/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql @@ -546,6 +546,8 @@ select fn_db_add_config_value('VmPoolMonitorIntervalInMinutes','5','general'); select fn_db_add_config_value('VmPoolMonitorMaxAttempts','3','general'); select fn_db_add_config_value('VmPriorityMaxValue','100','general'); +--How often we'll try to run HA VM that we couldn't run before +select fn_db_add_config_value('AutoStartVmsRunnerIntervalInSeconds','60','general'); --Handling Keyboard Layout configuration for VNC select fn_db_add_config_value('VncKeyboardLayout','en-us','general'); select fn_db_add_config_value('VncKeyboardLayoutValidValues','ar,da,de,de-ch,en-gb,en-us,es,et,fi,fo,fr,fr-be,fr-ca,fr-ch,hr,hu,is,it,ja,lt,lv,mk,nl,nl-be,no,pl,pt,pt-br,ru,sl,sv,th,tr','general'); -- To view, visit http://gerrit.ovirt.org/18815 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3d563d05efc6dae40f0de8e7f4b7e47aa84bd787 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Arik Hadas <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
