Arik Hadas has uploaded a new change for review. Change subject: core: rerun HA vm that fails during live snapshot ......................................................................
core: rerun HA vm that fails during live snapshot This patch fix a bug that happens when HA vm fails during live snapshot operation, and stays down. The problem was that when the monitoring detected that the vm is dowm, it tried to rerun it, but the RunVm command failed because the vm was during snapshot. The solution is to save the VM status when starting to execute the CreateAllSnapshots command, and at the end check if the VM is HA + it's down because of an error + it was up when the command started - that would mean the VM went down unexpectedly during the snapshot command, and thus will try to run it. Change-Id: Id5daeffd1cee106019065c29fb8428a9b9f78a5d Bug-Url: https://bugzilla.redhat.com/878041 Signed-off-by: Arik Hadas <[email protected]> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/CreateAllSnapshotsFromVmCommand.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/CreateAllSnapshotsFromVmParameters.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/ResourceManager.java M backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java 4 files changed, 63 insertions(+), 24 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/18/10618/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/CreateAllSnapshotsFromVmCommand.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/CreateAllSnapshotsFromVmCommand.java index 703cb4e..102937c 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/CreateAllSnapshotsFromVmCommand.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/CreateAllSnapshotsFromVmCommand.java @@ -17,14 +17,17 @@ import org.ovirt.engine.core.common.VdcObjectType; import org.ovirt.engine.core.common.action.CreateAllSnapshotsFromVmParameters; import org.ovirt.engine.core.common.action.ImagesActionsParametersBase; +import org.ovirt.engine.core.common.action.RunVmParams; import org.ovirt.engine.core.common.action.VdcActionParametersBase; import org.ovirt.engine.core.common.action.VdcActionType; import org.ovirt.engine.core.common.action.VdcReturnValueBase; import org.ovirt.engine.core.common.businessentities.Disk; import org.ovirt.engine.core.common.businessentities.DiskImage; +import org.ovirt.engine.core.common.businessentities.VMStatus; import org.ovirt.engine.core.common.businessentities.Snapshot.SnapshotStatus; import org.ovirt.engine.core.common.businessentities.Snapshot.SnapshotType; import org.ovirt.engine.core.common.businessentities.VM; +import org.ovirt.engine.core.common.businessentities.VmExitStatus; import org.ovirt.engine.core.common.config.Config; import org.ovirt.engine.core.common.config.ConfigValues; import org.ovirt.engine.core.common.errors.VdcBLLException; @@ -44,7 +47,7 @@ @DisableInPrepareMode @LockIdNameAttribute -public class CreateAllSnapshotsFromVmCommand<T extends CreateAllSnapshotsFromVmParameters> extends VmCommand<T> implements QuotaStorageDependent{ +public class CreateAllSnapshotsFromVmCommand<T extends CreateAllSnapshotsFromVmParameters> extends VmCommand<T> implements QuotaStorageDependent { private static final long serialVersionUID = -2407757772735253053L; List<DiskImage> selectedActiveDisks; @@ -69,7 +72,6 @@ return jobProperties; } - /** * Filter all allowed snapshot disks. * @return list of disks to be snapshot. @@ -88,6 +90,7 @@ Guid newActiveSnapshotId = Guid.NewGuid(); Guid createdSnapshotId = getSnapshotDao().getId(getVmId(), SnapshotType.ACTIVE); getParameters().setSnapshotType(determineSnapshotType()); + getParameters().setInitialVmStatus(getVm().getStatus()); getSnapshotDao().updateId(createdSnapshotId, newActiveSnapshotId); new SnapshotsManager().addSnapshot(createdSnapshotId, @@ -145,25 +148,49 @@ @Override protected void endVmCommand() { - Guid createdSnapshotId = - getSnapshotDao().getId(getVmId(), getParameters().getSnapshotType(), SnapshotStatus.LOCKED); - if (getParameters().getTaskGroupSuccess()) { - getSnapshotDao().updateStatus(createdSnapshotId, SnapshotStatus.OK); + TransactionSupport.executeInNewTransaction(new TransactionMethod<Void>() { - if (getParameters().getParentCommand() != VdcActionType.RunVm && getVm() != null && getVm().isStatusUp() - && getVm().getRunOnVds() != null) { - performLiveSnapshot(createdSnapshotId); + @Override + public Void runInTransaction() { + final boolean taskGroupSucceeded = getParameters().getTaskGroupSuccess(); + Guid createdSnapshotId = + getSnapshotDao().getId(getVmId(), getParameters().getSnapshotType(), SnapshotStatus.LOCKED); + if (taskGroupSucceeded) { + getSnapshotDao().updateStatus(createdSnapshotId, SnapshotStatus.OK); + + if (getParameters().getParentCommand() != VdcActionType.RunVm && getVm() != null && getVm().isStatusUp() + && getVm().getRunOnVds() != null) { + performLiveSnapshot(createdSnapshotId); + } + } else { + revertToActiveSnapshot(createdSnapshotId); + } + + getVmStaticDAO().incrementDbGeneration(getVm().getId()); + + endActionOnDisks(); + setSucceeded(taskGroupSucceeded); + getReturnValue().setEndActionTryAgain(false); + return null; } - } else { - revertToActiveSnapshot(createdSnapshotId); + }); + + if (isVmDownUnexpectedly() && getVm().isAutoStartup() && wasVmUpInitially()) { + Backend.getInstance().runInternalAction(VdcActionType.RunVm, + new RunVmParams(getVmId()), + ExecutionHandler.createInternalJobContext()); } + } - getVmStaticDAO().incrementDbGeneration(getVm().getId()); + private boolean isVmDownUnexpectedly() { + VM vm = getVmDAO().get(getVmId()); + return ImagesHandler.isVmDown(vm) && getVm().getExitStatus() != VmExitStatus.Normal; + } - endActionOnDisks(); - - setSucceeded(getParameters().getTaskGroupSuccess()); - getReturnValue().setEndActionTryAgain(false); + private boolean wasVmUpInitially() { + VMStatus vmStatus = getParameters().getInitialVmStatus(); + return vmStatus == VMStatus.Up || vmStatus == VMStatus.PoweringUp || vmStatus == VMStatus.MigratingFrom + || vmStatus == VMStatus.MigratingTo || vmStatus == VMStatus.WaitForLaunch; } /** @@ -286,8 +313,6 @@ return Config.<Boolean> GetValue( ConfigValues.LiveSnapshotEnabled, getStoragePool().getcompatibility_version().getValue()); } - - @Override protected VdcActionType getChildActionType() { diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/CreateAllSnapshotsFromVmParameters.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/CreateAllSnapshotsFromVmParameters.java index beb4971..f6cc9eb 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/CreateAllSnapshotsFromVmParameters.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/CreateAllSnapshotsFromVmParameters.java @@ -5,6 +5,7 @@ import org.hibernate.validator.constraints.NotEmpty; import org.ovirt.engine.core.common.businessentities.BusinessEntitiesDefinitions; import org.ovirt.engine.core.common.businessentities.Snapshot.SnapshotType; +import org.ovirt.engine.core.common.businessentities.VMStatus; import org.ovirt.engine.core.common.validation.annotation.ValidDescription; import org.ovirt.engine.core.common.validation.group.CreateEntity; import org.ovirt.engine.core.compat.Guid; @@ -20,10 +21,10 @@ private String _description; private boolean needsLocking = true; + /** Used to store the vm status when the command start, will be used to check if the vm went down during the execution */ + private VMStatus initialVmStatus; - /** - * Used to indicate the type of snapshot to take. - */ + /** Used to indicate the type of snapshot to take */ private SnapshotType snapshotType; public CreateAllSnapshotsFromVmParameters() { @@ -42,6 +43,14 @@ return snapshotType; } + public void setInitialVmStatus(VMStatus vmStatus) { + this.initialVmStatus = vmStatus; + } + + public VMStatus getInitialVmStatus() { + return initialVmStatus; + } + /** * This method is for internal use only, disregard in API. * @param snapshotType diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/ResourceManager.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/ResourceManager.java index 6a3cb75..d69c81f 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/ResourceManager.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/ResourceManager.java @@ -15,6 +15,7 @@ import org.ovirt.engine.core.common.businessentities.VMStatus; import org.ovirt.engine.core.common.businessentities.VdsDynamic; import org.ovirt.engine.core.common.businessentities.VdsStatistics; +import org.ovirt.engine.core.common.businessentities.VmExitStatus; import org.ovirt.engine.core.common.businessentities.VmPauseStatus; import org.ovirt.engine.core.common.businessentities.network.NetworkStatistics; import org.ovirt.engine.core.common.businessentities.network.VmNetworkInterface; @@ -241,6 +242,8 @@ */ public void InternalSetVmStatus(VM vm, final VMStatus status) { vm.setStatus(status); + vm.setExitStatus(VmExitStatus.Normal); + vm.setExitMessage(StringUtils.EMPTY); boolean isVmStatusDown = VM.isStatusDown(status); if (isVmStatusDown || status == VMStatus.Unknown) { diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java index d9f1077..1076780 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsUpdateRunTimeInfo.java @@ -1178,7 +1178,7 @@ ResourceManager.getInstance().InternalSetVmStatus(vmTo, VMStatus.Suspended); } - clearVm(vmTo); + clearVm(vmTo, vmInternalData.getVmDynamic().getExitMessage()); } VmStatistics vmStatistics = getDbFacade().getVmStatisticsDao().get(vm.getId()); @@ -1451,7 +1451,7 @@ AddVmStatisticsToList(vmToRemove.getStatisticsData()); AddVmInterfaceStatisticsToList(vmToRemove.getInterfaces()); } else { - clearVm(vmToRemove); + clearVm(vmToRemove, "VM disappeared"); } log.infoFormat("vm {0} running in db and not running in vds - add to rerun treatment. vds {1}", vmToRemove.getVmName(), _vds.getvds_name()); @@ -1729,10 +1729,12 @@ vmDeviceToSave.put(vmDevice.getId(), vmDevice); } - private void clearVm(VM vm) { + private void clearVm(VM vm, String exitMessage) { if (vm.getStatus() != VMStatus.MigratingFrom) { if (vm.getStatus() != VMStatus.Suspended) { ResourceManager.getInstance().InternalSetVmStatus(vm, VMStatus.Down); + vm.setExitStatus(VmExitStatus.Error); + vm.setExitMessage(exitMessage); } AddVmDynamicToList(vm.getDynamicData()); AddVmStatisticsToList(vm.getStatisticsData()); -- To view, visit http://gerrit.ovirt.org/10618 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Id5daeffd1cee106019065c29fb8428a9b9f78a5d Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Arik Hadas <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
