Martin Peřina has uploaded a new change for review. Change subject: core: Introduce FenceAgentExecutor ......................................................................
core: Introduce FenceAgentExecutor Introduces FenceAgentExecutor which is responsible to execute fence action on one agent. It cooperates with FenceProxyLocator and uses the retry logic if 1st fence operation failed. Change-Id: I10936f4bc6a3b824a10a7ebc39de1f3a48eb778e Bug-Url: https://bugzilla.redhat.com/1182510 Signed-off-by: Martin Perina <[email protected]> --- A backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/FenceAgentExecutor.java A backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/FenceAgentExecutorTest.java 2 files changed, 358 insertions(+), 0 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/63/38063/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/FenceAgentExecutor.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/FenceAgentExecutor.java new file mode 100644 index 0000000..39fdc94 --- /dev/null +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/FenceAgentExecutor.java @@ -0,0 +1,168 @@ +package org.ovirt.engine.core.bll.pm; + +import org.ovirt.engine.core.bll.FenceProxyLocator; +import org.ovirt.engine.core.common.businessentities.ArchitectureType; +import org.ovirt.engine.core.common.businessentities.FenceAgent; +import org.ovirt.engine.core.common.businessentities.FencingPolicy; +import org.ovirt.engine.core.common.businessentities.VDS; +import org.ovirt.engine.core.common.businessentities.pm.FenceActionType; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult.Status; +import org.ovirt.engine.core.common.businessentities.pm.HostPowerStatus; +import org.ovirt.engine.core.common.errors.VdcBLLException; +import org.ovirt.engine.core.common.vdscommands.FenceVdsVDSCommandParameters; +import org.ovirt.engine.core.common.vdscommands.VDSCommandType; +import org.ovirt.engine.core.common.vdscommands.VDSReturnValue; +import org.ovirt.engine.core.utils.pm.VdsFenceOptions; +import org.ovirt.engine.core.vdsbroker.ResourceManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages fence execution of single agent for the host + */ +public class FenceAgentExecutor { + private static final Logger log = LoggerFactory.getLogger(FenceAgentExecutor.class); + + private final VDS vds; + private FencingPolicy fencingPolicy; + private FenceProxyLocator proxyLocator; + private ArchitectureType architectureType; + + public FenceAgentExecutor( + VDS vds, + FenceProxyLocator locator, + ArchitectureType architectureType, + FencingPolicy fencingPolicy) { + /* + TODO: Move to PmActionExecutor + TODO remove if block after UI patch that should set also cluster & proxy preferences in GetNewVdsFenceStatusParameters + if (! vds.getId().equals(Guid.Empty)) { + VDS dbVds = DbFacade.getInstance().getVdsDao().get(vds.getId()); + if (vds.getVdsGroupId() == null) { + vds.setVdsGroupId(dbVds.getVdsGroupId()); + } + if (vds.getPmProxyPreferences() == null) { + vds.setPmProxyPreferences(dbVds.getPmProxyPreferences()); + } + } + */ + this.vds = vds; + this.proxyLocator = locator; + this.architectureType = architectureType; + this.fencingPolicy = fencingPolicy; + } + + public FenceOperationResult fence(FenceActionType action, FenceAgent agent) { + boolean withRetries = action != FenceActionType.STATUS; + VDS proxyHost = proxyLocator.findProxyHost(withRetries); + if (proxyHost == null) { + return new FenceOperationResult( + Status.ERROR, + HostPowerStatus.UNKNOWN, + "Failed to run Power Management command on Host '" + getNameOrId(vds) + "' no proxy was found"); + } else { + return fence(action, agent, proxyHost); + } + } + + public FenceOperationResult fence(FenceActionType action, FenceAgent agent, VDS proxyHost) { + FenceOperationResult result = null; + try { + /* + TODO: Move to PmActionExecutor + if (action == FenceActionType.Restart || action == FenceActionType.Stop) { + stopSPM(action); + } + */ + result = runFenceAction(action, agent, proxyHost); + // if fence failed, retry with another proxy. + if (result.getStatus() == Status.ERROR) { + log.warn("Fence operation failed with proxy host '{}', trying another proxy", + proxyHost.getId()); + boolean withRetries = action != FenceActionType.STATUS; + VDS alternativeProxy = proxyLocator.findProxyHost(withRetries, proxyHost.getId()); + if (alternativeProxy != null) { + result = runFenceAction(action, agent, alternativeProxy); + } else { + log.warn("Failed to find another proxy to re-run failed fence operation, " + + "retrying with the same proxy '{}'", + proxyHost); + result = runFenceAction(action, agent, proxyHost); + } + } + } catch (VdcBLLException e) { + log.debug("Exception", e); + result = new FenceOperationResult( + FenceOperationResult.Status.ERROR, + HostPowerStatus.UNKNOWN, + e.getMessage()); + } + return result; + } + + /** + * Run the specified fence action. + */ + private FenceOperationResult runFenceAction(FenceActionType action, FenceAgent agent, VDS proxyHost) { + log.debug("Executing '{}' PM command, proxy: '{}', agent: '{}', type: '{}', target: '{}', ip: '{}'," + + " user: '{}', options: '{}', policy: '{}'", + action, + getNameOrId(proxyHost), + agent.getId(), + VdsFenceOptions.getRealAgent(agent.getType()), + getNameOrId(vds), + agent.getIp(), + agent.getUser(), + getOptions(agent), + fencingPolicy); + + VDSReturnValue retVal = getResourceManager().runVdsCommand( + VDSCommandType.FenceVds, + new FenceVdsVDSCommandParameters( + proxyHost.getId(), + vds.getId(), + agent, + action, + fencingPolicy)); + + FenceOperationResult result = (FenceOperationResult) retVal.getReturnValue(); + if (result == null) { + log.error( + "FenceVdsVDSCommand finished with null return value: succeeded={}, exceptionString='{}'", + retVal.getSucceeded(), + retVal.getExceptionString()); + log.debug("Exception", retVal.getExceptionObject()); + result = new FenceOperationResult( + Status.ERROR, + HostPowerStatus.UNKNOWN, + retVal.getExceptionString()); + } + return result; + } + + /** + * Merges agent specific options with default options for architecture and convert them to string + */ + private String getOptions(FenceAgent agent) { + return VdsFenceOptions.getDefaultAgentOptions( + agent.getType(), + agent.getOptions() == null ? "" : agent.getOptions(), + architectureType); + } + + /** + * Return host name if available or host UUID if not + */ + private String getNameOrId(VDS host) { + if (host.getName() != null && !host.getName().isEmpty()) { + return host.getName(); + } else { + return host.getId().toString(); + } + } + + ResourceManager getResourceManager() { + return ResourceManager.getInstance(); + } +} diff --git a/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/FenceAgentExecutorTest.java b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/FenceAgentExecutorTest.java new file mode 100644 index 0000000..0b576c1 --- /dev/null +++ b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/FenceAgentExecutorTest.java @@ -0,0 +1,190 @@ +package org.ovirt.engine.core.bll.pm; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.stub; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.ovirt.engine.core.bll.FenceProxyLocator; +import org.ovirt.engine.core.common.businessentities.ArchitectureType; +import org.ovirt.engine.core.common.businessentities.FenceAgent; +import org.ovirt.engine.core.common.businessentities.FencingPolicy; +import org.ovirt.engine.core.common.businessentities.VDS; +import org.ovirt.engine.core.common.businessentities.pm.FenceActionType; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult; +import org.ovirt.engine.core.common.businessentities.pm.FenceOperationResult.Status; +import org.ovirt.engine.core.common.config.ConfigValues; +import org.ovirt.engine.core.common.vdscommands.FenceVdsVDSCommandParameters; +import org.ovirt.engine.core.common.vdscommands.VDSCommandType; +import org.ovirt.engine.core.common.vdscommands.VDSReturnValue; +import org.ovirt.engine.core.compat.Guid; +import org.ovirt.engine.core.compat.Version; +import org.ovirt.engine.core.utils.MockConfigRule; +import org.ovirt.engine.core.vdsbroker.ResourceManager; + +@RunWith(MockitoJUnitRunner.class) +public class FenceAgentExecutorTest { + + private static final Version CLUSTER_VERSION = Version.v3_0; + private static Guid FENCECD_HOST_ID = new Guid("11111111-1111-1111-1111-111111111111"); + private static Guid PROXY_HOST_ID = new Guid("44444444-4444-4444-4444-444444444444"); + private static Guid SECOND_PROXY_HOST_ID = new Guid("77777777-7777-7777-7777-777777777777"); + private static Guid FENCE_AGENT_ID = new Guid("55555555-5555-5555-5555-555555555555"); + + @ClassRule + public static MockConfigRule configRule = + new MockConfigRule(MockConfigRule.mockConfig(ConfigValues.FenceAgentMapping, "")); + + @Mock + private VDS vds; + + @Mock + private FenceProxyLocator proxyLocator; + + private FenceAgentExecutor executor; + + private FencingPolicy fencingPolicy; + + @Mock + private ResourceManager resourceManager; + + @Before + public void setup() { + mockVds(); + fencingPolicy = new FencingPolicy(); + executor = new FenceAgentExecutor(vds, proxyLocator, ArchitectureType.x86_64, fencingPolicy); + executor = spy(executor); + stub(executor.getResourceManager()).toReturn(resourceManager); + } + + private void mockVds() { + when(vds.getId()).thenReturn(FENCECD_HOST_ID); + } + + private void mockProxyHost() { + mockProxyHost(false); + } + + private void mockProxyHost(boolean anotherProxyAvailable) { + VDS proxyHost = new VDS(); + proxyHost.setId(PROXY_HOST_ID); + when(proxyLocator.findProxyHost()).thenReturn(proxyHost); + when(proxyLocator.findProxyHost(true)).thenReturn(proxyHost); + VDS secondProxyHost = new VDS(); + if (anotherProxyAvailable) { + secondProxyHost.setId(SECOND_PROXY_HOST_ID); + when(proxyLocator.findProxyHost(true, PROXY_HOST_ID)).thenReturn(secondProxyHost); + } else { + when(proxyLocator.findProxyHost(true, PROXY_HOST_ID)).thenReturn(null); + } + } + + private VDSReturnValue createVdsReturnValue(FenceOperationResult result) { + VDSReturnValue retVal = new VDSReturnValue(); + retVal.setSucceeded(result.getStatus() != Status.ERROR); + retVal.setReturnValue(result); + return retVal; + } + + private void mockFenceVdsResult(FenceOperationResult result1, FenceOperationResult result2) { + VDSReturnValue retVal1 = createVdsReturnValue(result1); + VDSReturnValue retVal2 = result2 == null ? null : createVdsReturnValue(result2); + when(resourceManager.runVdsCommand( + eq(VDSCommandType.FenceVds), + any(FenceVdsVDSCommandParameters.class))) + .thenReturn(retVal1) + .thenReturn(retVal2); + } + + private FenceAgent createAgent() { + FenceAgent agent = new FenceAgent(); + agent.setId(FENCE_AGENT_ID); + return agent; + } + + /** + * Test that the return value is correct when fencing succeeds. The return value should contain succeeded=true and + * the agent used. + */ + @Test + public void successfulFence() { + FenceOperationResult fenceVdsResult = new FenceOperationResult(Status.SUCCESS); + mockFenceVdsResult(fenceVdsResult, null); + mockProxyHost(); + + FenceOperationResult result = executor.fence(FenceActionType.START, createAgent()); + assertEquals(Status.SUCCESS, result.getStatus()); + } + + /** + * Test that when first fence attempt fails, fence is retried with a different proxy. + */ + @Test + public void successfulFenceWithDifferentProxyRetry() { + FenceOperationResult fenceVdsResult1 = new FenceOperationResult(Status.ERROR); + FenceOperationResult fenceVdsResult2 = new FenceOperationResult(Status.SUCCESS); + mockFenceVdsResult(fenceVdsResult1, fenceVdsResult2); + mockProxyHost(true); + + FenceOperationResult result = executor.fence(FenceActionType.START, createAgent()); + assertEquals(Status.SUCCESS, result.getStatus()); + verify(proxyLocator).findProxyHost(true, PROXY_HOST_ID); + } + + /** + * Test that when first fence attempt fails, and no alternative proxy is found, fence is retried with the same + * proxy. + */ + @Test + public void successfulFenceWithSameProxyRetry() { + FenceOperationResult fenceVdsResult1 = new FenceOperationResult(Status.ERROR); + FenceOperationResult fenceVdsResult2 = new FenceOperationResult(Status.SUCCESS); + mockFenceVdsResult(fenceVdsResult1, fenceVdsResult2); + mockProxyHost(false); + + FenceOperationResult result = executor.fence(FenceActionType.START, createAgent()); + assertEquals(Status.SUCCESS, result.getStatus()); + verify(proxyLocator).findProxyHost(true, PROXY_HOST_ID); + } + + /** + * Test that when first fence attempt fails, and also the second attempt using a different proxy fails, then + * the whole fence execution fails + */ + @Test + public void failedFenceWithDifferentProxyRetry() { + FenceOperationResult fenceVdsResult1 = new FenceOperationResult(Status.ERROR); + FenceOperationResult fenceVdsResult2 = new FenceOperationResult(Status.ERROR); + mockFenceVdsResult(fenceVdsResult1, fenceVdsResult2); + mockProxyHost(true); + + FenceOperationResult result = executor.fence(FenceActionType.START, createAgent()); + assertEquals(Status.ERROR, result.getStatus()); + verify(proxyLocator).findProxyHost(true, PROXY_HOST_ID); + } + + /** + * Test that when first fence attempt fails, and also the second attempt using the same proxy fails, then + * the whole fence execution fails + */ + @Test + public void failedFenceWithSameProxyRetry() { + FenceOperationResult fenceVdsResult1 = new FenceOperationResult(Status.ERROR); + FenceOperationResult fenceVdsResult2 = new FenceOperationResult(Status.ERROR); + mockFenceVdsResult(fenceVdsResult1, fenceVdsResult2); + mockProxyHost(false); + + FenceOperationResult result = executor.fence(FenceActionType.START, createAgent()); + assertEquals(Status.ERROR, result.getStatus()); + verify(proxyLocator).findProxyHost(true, PROXY_HOST_ID); + } +} -- To view, visit http://gerrit.ovirt.org/38063 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I10936f4bc6a3b824a10a7ebc39de1f3a48eb778e Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Martin Peřina <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
