Michael Kublin has uploaded a new change for review. Change subject: engine: if connect storage pool fails on version mismatch, do reconstruct master ......................................................................
engine: if connect storage pool fails on version mismatch, do reconstruct master The following patch will allow to perform reconstruct from InitVdsOnUp, for case that we can not connect to pool because of wrong master version or missing master domain. Using of event queue will protect from parallel reconstructs and from other races. The flow is a following: 1. Try connect to pool 2. If failed on wrong master domain or its version - do reconstruct 3. If success move host to UP Please note: I remove an update of vms at spm after reconstruct, because of it will be removed at the future and that code is actually failed a one host scenario. Change-Id: Ia0d11c068399664daf85a6b03e67d25cd382707e Bug-Url: https://bugzilla.redhat.com/882837 Signed-off-by: Michael Kublin <[email protected]> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/InitVdsOnUpCommand.java M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/eventqueue/EventQueueMonitor.java M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/ConnectHostToStoragePoolServersCommand.java M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/ReconstructMasterDomainCommand.java M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/StorageHandlingCommandBase.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/ReconstructMasterParameters.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventType.java M backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogableBase.java 8 files changed, 101 insertions(+), 52 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/03/10103/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/InitVdsOnUpCommand.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/InitVdsOnUpCommand.java index 6b81d69..c0f0e04 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/InitVdsOnUpCommand.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/InitVdsOnUpCommand.java @@ -5,11 +5,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; import org.ovirt.engine.core.bll.job.ExecutionHandler; import org.ovirt.engine.core.bll.storage.StorageHandlingCommandBase; import org.ovirt.engine.core.bll.storage.StoragePoolStatusHandler; import org.ovirt.engine.core.common.AuditLogType; +import org.ovirt.engine.core.common.action.ReconstructMasterParameters; import org.ovirt.engine.core.common.action.SetNonOperationalVdsParameters; import org.ovirt.engine.core.common.action.StoragePoolParametersBase; import org.ovirt.engine.core.common.action.VdcActionType; @@ -25,6 +27,11 @@ import org.ovirt.engine.core.common.businessentities.storage_pool; import org.ovirt.engine.core.common.businessentities.gluster.GlusterServerInfo; import org.ovirt.engine.core.common.errors.VdcBLLException; +import org.ovirt.engine.core.common.errors.VdcBllErrors; +import org.ovirt.engine.core.common.eventqueue.Event; +import org.ovirt.engine.core.common.eventqueue.EventQueue; +import org.ovirt.engine.core.common.eventqueue.EventResult; +import org.ovirt.engine.core.common.eventqueue.EventType; import org.ovirt.engine.core.common.vdscommands.ConnectStoragePoolVDSCommandParameters; import org.ovirt.engine.core.common.vdscommands.VDSCommandType; import org.ovirt.engine.core.common.vdscommands.VDSReturnValue; @@ -37,6 +44,9 @@ import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector; import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableBase; import org.ovirt.engine.core.dao.InterfaceDAO; +import org.ovirt.engine.core.utils.ejb.BeanProxyType; +import org.ovirt.engine.core.utils.ejb.BeanType; +import org.ovirt.engine.core.utils.ejb.EjbUtils; import org.ovirt.engine.core.vdsbroker.irsbroker.IrsBrokerCommand; /** @@ -56,7 +66,6 @@ public InitVdsOnUpCommand(T parameters) { super(parameters); - setVdsId(parameters.getVdsId()); } @Override @@ -133,30 +142,73 @@ .runInternalAction(VdcActionType.ConnectHostToStoragePoolServers, tempStorageBaseParams) .getSucceeded()) { _connectStorageSucceeded = true; - try { - setStoragePool(null); - returnValue = _connectPoolSucceeded = runVdsCommand( - VDSCommandType.ConnectStoragePool, - new ConnectStoragePoolVDSCommandParameters(getVds().getId(), getVds() - .getStoragePoolId(), getVds().getvds_spm_id(), getMasterDomainIdFromDb(), - getStoragePool().getmaster_domain_version())).getSucceeded(); - } catch (RuntimeException exp) { - log.errorFormat("Could not connect host {0} to pool {1}", getVds().getvds_name(), getStoragePool() - .getname()); - returnValue = false; - } - if(returnValue) { - returnValue = proceedVdsStats(); - if(!returnValue) { - AuditLogDirector.log(new AuditLogableBase(getVdsId()), - AuditLogType.VDS_STORAGE_VDS_STATS_FAILED); - } - } + returnValue = connectHostToPool(); + _connectPoolSucceeded = returnValue; } } return returnValue; } + /** + * The following method should connect host to pool + * The method will perform a connect storage pool operation, + * if operation will wail on StoragePoolWrongMaster or StoragePoolMasterNotFound errors + * we will try to run reconstruct + * @return + */ + private boolean connectHostToPool() { + setStoragePool(null); + final Guid masterDomainIdFromDb = getMasterDomainIdFromDb(); + final VDS vds = getVds(); + final storage_pool storagePool = getStoragePool(); + EventResult result = + ((EventQueue) EjbUtils.findBean(BeanType.EVENTQUEUE_MANAGER, BeanProxyType.LOCAL)).submitEventSync(new Event(storagePool.getId(), + masterDomainIdFromDb, vds.getId(), EventType.VDSCONNECTTOPOOL), + new Callable<EventResult>() { + @Override + public EventResult call() { + return runConnectHostToPoolEvent(masterDomainIdFromDb, vds, storagePool); + } + }); + return result.isSuccess(); + } + + private EventResult runConnectHostToPoolEvent(final Guid masterDomainIdFromDb, final VDS vds, final storage_pool storagePool) { + EventResult result = new EventResult(true, EventType.VDSCONNECTTOPOOL); + try { + runVdsCommand(VDSCommandType.ConnectStoragePool, + new ConnectStoragePoolVDSCommandParameters(vds.getId(), vds + .getStoragePoolId(), vds.getvds_spm_id(), masterDomainIdFromDb, + storagePool.getmaster_domain_version())).getSucceeded(); + } catch (VdcBLLException e) { + if (e.getErrorCode() == VdcBllErrors.StoragePoolWrongMaster + || e.getErrorCode() == VdcBllErrors.StoragePoolMasterNotFound) { + boolean returnValue = + Backend.getInstance() + .runInternalAction(VdcActionType.ReconstructMasterDomain, + new ReconstructMasterParameters(vds.getStoragePoolId(), + masterDomainIdFromDb, vds.getId(), false)).getSucceeded(); + result = new EventResult(returnValue, EventType.RECONSTRUCT); + } else { + log.errorFormat("Could not connect host {0} to pool {1}", vds.getvds_name(), storagePool + .getname()); + result.setSuccess(false); + } + } catch (RuntimeException exp) { + log.errorFormat("Could not connect host {0} to pool {1}", vds.getvds_name(), storagePool + .getname()); + result.setSuccess(false); + } + if(result.isSuccess() && result.getEventType() != EventType.RECONSTRUCT) { + result.setSuccess(proceedVdsStats()); + if(!result.isSuccess()) { + AuditLogDirector.log(new AuditLogableBase(getVdsId()), + AuditLogType.VDS_STORAGE_VDS_STATS_FAILED); + } + } + return result; + } + protected boolean proceedVdsStats() { boolean returnValue = true; try { diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/eventqueue/EventQueueMonitor.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/eventqueue/EventQueueMonitor.java index 362c91b..80c3553 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/eventqueue/EventQueueMonitor.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/eventqueue/EventQueueMonitor.java @@ -151,14 +151,24 @@ log.infoFormat("Finished reconstruct for pool {0}. Clearing all event queue", storagePoolId); lock.lock(); try { + Queue<Pair<Event, FutureTask<EventResult>>> queue = + new LinkedList<Pair<Event, FutureTask<EventResult>>>(); for (Pair<Event, FutureTask<EventResult>> task : poolsEventsMap.get(storagePoolId)) { - log.infoFormat("The following operation {0} was cancelled, because of recosntruct was run before", - task.getFirst()); - task.getSecond().cancel(true); + if (task.getFirst().getEventType() != EventType.VDSCONNECTTOPOOL) { + log.infoFormat("The following operation {0} was cancelled, because of recosntruct was run before", + task.getFirst()); + task.getSecond().cancel(true); + } else { + queue.add(task); + } } - poolCurrentEventMap.remove(storagePoolId); - poolsEventsMap.remove(storagePoolId); - break; + if (queue.isEmpty()) { + poolCurrentEventMap.remove(storagePoolId); + poolsEventsMap.remove(storagePoolId); + break; + } else { + poolsEventsMap.put(storagePoolId, queue); + } } finally { lock.unlock(); } diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/ConnectHostToStoragePoolServersCommand.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/ConnectHostToStoragePoolServersCommand.java index 2f648e3..aa1b3a8 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/ConnectHostToStoragePoolServersCommand.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/ConnectHostToStoragePoolServersCommand.java @@ -24,7 +24,6 @@ public ConnectHostToStoragePoolServersCommand(T parameters) { super(parameters); - setVdsId(parameters.getVdsId()); } @Override diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/ReconstructMasterDomainCommand.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/ReconstructMasterDomainCommand.java index 2c3ce7d..4b926f6 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/ReconstructMasterDomainCommand.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/ReconstructMasterDomainCommand.java @@ -1,25 +1,19 @@ package org.ovirt.engine.core.bll.storage; -import java.text.MessageFormat; import java.util.List; import org.ovirt.engine.core.bll.Backend; import org.ovirt.engine.core.bll.NonTransactiveCommandAttribute; -import org.ovirt.engine.core.bll.VmCommand; import org.ovirt.engine.core.common.AuditLogType; import org.ovirt.engine.core.common.action.ReconstructMasterParameters; import org.ovirt.engine.core.common.businessentities.StorageDomainStatus; import org.ovirt.engine.core.common.businessentities.VDS; import org.ovirt.engine.core.common.businessentities.VDSStatus; -import org.ovirt.engine.core.common.businessentities.VM; import org.ovirt.engine.core.common.businessentities.storage_pool_iso_map; import org.ovirt.engine.core.common.config.Config; import org.ovirt.engine.core.common.config.ConfigValues; import org.ovirt.engine.core.common.errors.VdcBLLException; import org.ovirt.engine.core.common.errors.VdcBllErrors; -import org.ovirt.engine.core.common.interfaces.SearchType; -import org.ovirt.engine.core.common.queries.SearchParameters; -import org.ovirt.engine.core.common.queries.VdcQueryType; import org.ovirt.engine.core.common.vdscommands.ConnectStoragePoolVDSCommandParameters; import org.ovirt.engine.core.common.vdscommands.DisconnectStoragePoolVDSCommandParameters; import org.ovirt.engine.core.common.vdscommands.MarkPoolInReconstructModeVDSCommandParameters; @@ -140,21 +134,6 @@ try { reconstructOpSucceeded = reconstructMaster(); connectAndRefreshAllUpHosts(reconstructOpSucceeded); - - if (!_isLastMaster && reconstructOpSucceeded) { - SearchParameters p = new SearchParameters( - MessageFormat.format(DesktopsInStoragePoolQuery, - getStoragePool().getname()), SearchType.VM); - - p.setMaxCount(Integer.MAX_VALUE); - - @SuppressWarnings("unchecked") - List<VM> vmsInPool = (List<VM>) Backend.getInstance() - .runInternalQuery(VdcQueryType.Search, p).getReturnValue(); - - VmCommand.updateVmInSpm(getStoragePool().getId(), vmsInPool); - } - setSucceeded(reconstructOpSucceeded); } finally { // reset cache and mark reconstruct for pool as finished diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/StorageHandlingCommandBase.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/StorageHandlingCommandBase.java index 0631698..9153355 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/StorageHandlingCommandBase.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/StorageHandlingCommandBase.java @@ -39,6 +39,7 @@ public abstract class StorageHandlingCommandBase<T extends StoragePoolParametersBase> extends CommandBase<T> { public StorageHandlingCommandBase(T parameters) { super(parameters); + setVdsId(parameters.getVdsId()); if (getParameters() != null && !getParameters().getStoragePoolId().equals(Guid.Empty)) { setStoragePoolId(getParameters().getStoragePoolId()); } diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/ReconstructMasterParameters.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/ReconstructMasterParameters.java index 3dc1c9b..f74e2a6 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/ReconstructMasterParameters.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/action/ReconstructMasterParameters.java @@ -15,6 +15,12 @@ setInactive(isInactive); } + public ReconstructMasterParameters(Guid storagePoolId, Guid storageDomainId, Guid vdsId, boolean isInactive) { + super(storageDomainId, storagePoolId); + setInactive(isInactive); + setVdsId(vdsId); + } + public ReconstructMasterParameters(Guid storagePoolId, Guid newMasterDomainId) { this(storagePoolId, Guid.Empty, false); this.privateNewMasterDomainId = newMasterDomainId; diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventType.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventType.java index 57b9bf7..107a376 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventType.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/eventqueue/EventType.java @@ -6,5 +6,6 @@ DOMAINNOTOPERATIONAL, VDSSTOARGEPROBLEMS, DOMAINMONITORING, - VDSCLEARCACHE; + VDSCLEARCACHE, + VDSCONNECTTOPOOL; } diff --git a/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogableBase.java b/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogableBase.java index fbf63d2..f5db6bf 100644 --- a/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogableBase.java +++ b/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogableBase.java @@ -290,9 +290,10 @@ } protected VDS getVds() { - if (mVds == null && (mVdsId != null || (getVm() != null && getVm().getRunOnVds() != null))) { - if (mVdsId == null) { - mVdsId = new Guid(getVm().getRunOnVds().toString()); + if (mVds == null + && ((mVdsId != null && !Guid.Empty.equals(mVdsId)) || (getVm() != null && getVm().getRunOnVds() != null))) { + if (mVdsId == null || Guid.Empty.equals(mVdsId)) { + mVdsId = getVm().getRunOnVds().getValue(); } try { mVds = getVdsDAO().get(getVdsId()); -- To view, visit http://gerrit.ovirt.org/10103 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ia0d11c068399664daf85a6b03e67d25cd382707e Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Michael Kublin <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
