Martin Peřina has uploaded a new change for review. Change subject: core: Refactor FenceProxyLocator ......................................................................
core: Refactor FenceProxyLocator 1. Makes FenceProxyLocator more readable 2. Fixes bug when testing agents compatibility between proxy candidate and fenced host 3. Makes FenceProxyLocatorTest more readable 4. Adds more tests to FenceProxyLocatorTest Change-Id: I59d5b46273dbaeb2d73142f1256000e36a6cbdd8 Bug-Url: https://bugzilla.redhat.com/1182510 Signed-off-by: Martin Perina <[email protected]> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/FenceProxyLocator.java M backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/FenceProxyLocatorTest.java 2 files changed, 376 insertions(+), 254 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/64/39764/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/FenceProxyLocator.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/FenceProxyLocator.java index 2c9f6af..e892feb 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/FenceProxyLocator.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/pm/FenceProxyLocator.java @@ -2,6 +2,7 @@ import java.util.Iterator; import java.util.List; +import java.util.concurrent.TimeUnit; import org.ovirt.engine.core.common.businessentities.FenceAgent; import org.ovirt.engine.core.common.businessentities.FencingPolicy; @@ -16,10 +17,14 @@ import org.ovirt.engine.core.compat.Guid; import org.ovirt.engine.core.compat.Version; import org.ovirt.engine.core.dal.dbbroker.DbFacade; +import org.ovirt.engine.core.utils.ThreadUtils; import org.ovirt.engine.core.utils.pm.VdsFenceOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * It manages selection of fence proxy for specified host and fencing policy + */ public class FenceProxyLocator { private static final Logger log = LoggerFactory.getLogger(FenceProxyLocator.class); @@ -48,24 +53,20 @@ } public VDS findProxyHost(boolean withRetries, Guid excludedHostId) { - // make sure that loop is executed at least once , no matter what is the - // value in config - int retries = Math.max(Config.<Integer> getValue(ConfigValues.FindFenceProxyRetries), 1); - int delayInMs = 1000 * Config.<Integer> getValue(ConfigValues.FindFenceProxyDelayBetweenRetriesInSec); + int retries = getFindFenceProxyRetries(); + long delayInMs = getDelayBetweenRetries(); VDS proxyHost = null; // get PM Proxy preferences or use defaults if not defined for (FenceProxySourceType fenceProxySource : getFenceProxySources()) { - proxyHost = chooseBestProxy(fenceProxySource, excludedHostId); + proxyHost = selectBestProxy(fenceProxySource, excludedHostId); int count = 0; // If can not find a proxy host retry and delay between retries as configured. while (proxyHost == null && withRetries && count < retries) { - log.warn("Attempt {} to find fence proxy host failed...", ++count); - try { - Thread.sleep(delayInMs); - } catch (Exception e) { - log.error(e.getMessage()); - } - proxyHost = chooseBestProxy(fenceProxySource, excludedHostId); + log.warn("Attempt {} to find fence proxy for host '{}' failed...", + ++count, + fencedHost.getHostName()); + ThreadUtils.sleep((int) delayInMs); + proxyHost = selectBestProxy(fenceProxySource, excludedHostId); } if (proxyHost != null) { break; @@ -79,87 +80,138 @@ return proxyHost; } - private List<FenceProxySourceType> getFenceProxySources() { + protected List<FenceProxySourceType> getFenceProxySources() { List<FenceProxySourceType> fenceProxySources = fencedHost.getFenceProxySources(); if (fenceProxySources == null || fenceProxySources.isEmpty()) { - fenceProxySources = FenceProxySourceTypeHelper.parseFromString( - Config.<String>getValue(ConfigValues.FenceProxyDefaultPreferences)); + fenceProxySources = getDefaultFenceProxySources(); } return fenceProxySources; } - private VDS chooseBestProxy(FenceProxySourceType fenceProxySource, Guid excludedHostId) { - List<VDS> hosts = DbFacade.getInstance().getVdsDao().getAll(); - Version minSupportedVersion = null; - if (fencingPolicy != null) { - minSupportedVersion = FencingPolicyHelper.getMinimalSupportedVersion(fencingPolicy); - } - Iterator<VDS> iterator = hosts.iterator(); + protected VDS selectBestProxy(FenceProxySourceType fenceProxySource, Guid excludedHostId) { + Version minSupportedVersion = getMinSupportedVersionForFencingPolicy(); + List<VDS> proxyCandidates = getDbFacade().getVdsDao().getAll(); + Iterator<VDS> iterator = proxyCandidates.iterator(); while (iterator.hasNext()) { - VDS host = iterator.next(); - if (host.getId().equals(fencedHost.getId()) - || host.getId().equals(excludedHostId) - || !matchesOption(host, fenceProxySource) - || !areAgentsVersionCompatible(host) - || (fencingPolicy != null && !isFencingPolicySupported(host, minSupportedVersion)) - || isHostNetworkUnreachable(host)) { + VDS proxyCandidate = iterator.next(); + log.debug("Evaluating host '{}'", proxyCandidate.getHostName()); + if (proxyCandidate.getId().equals(fencedHost.getId()) + || isHostExcluded(proxyCandidate, excludedHostId) + || !isHostFromSelectedSource(proxyCandidate, fenceProxySource) + || !areAgentsVersionCompatible(proxyCandidate) + || !isFencingPolicySupported(proxyCandidate, minSupportedVersion) + || isHostNetworkUnreachable(proxyCandidate)) { iterator.remove(); } } - for (VDS host : hosts) { - if (host.getStatus() == VDSStatus.Up) { - return host; + for (VDS proxyCandidate : proxyCandidates) { + if (proxyCandidate.getStatus() == VDSStatus.Up) { + return proxyCandidate; } } - return hosts.size() == 0 ? null : hosts.get(0); + return proxyCandidates.size() == 0 ? null : proxyCandidates.get(0); } - private boolean matchesOption(VDS proxyCandidate, FenceProxySourceType fenceProxySource) { - boolean matches = false; + protected boolean isHostExcluded(VDS proxyCandidate, Guid excludedHostId) { + boolean excluded = proxyCandidate.getId().equals(excludedHostId); + + log.debug("Proxy candidate '{}' was excluded intentionally: {}", + proxyCandidate.getHostName(), + excluded); + return excluded; + } + + private boolean isHostFromSelectedSource(VDS proxyCandidate, FenceProxySourceType fenceProxySource) { + boolean fromSelectedSource = false; switch (fenceProxySource) { case CLUSTER: - matches = proxyCandidate.getVdsGroupId().equals(fencedHost.getVdsGroupId()); + fromSelectedSource = proxyCandidate.getVdsGroupId().equals(fencedHost.getVdsGroupId()); break; case DC: - matches = proxyCandidate.getStoragePoolId().equals(fencedHost.getStoragePoolId()); + fromSelectedSource = proxyCandidate.getStoragePoolId().equals(fencedHost.getStoragePoolId()); break; case OTHER_DC: - matches = !proxyCandidate.getStoragePoolId().equals(fencedHost.getStoragePoolId()); + fromSelectedSource = !proxyCandidate.getStoragePoolId().equals(fencedHost.getStoragePoolId()); break; } - return matches; + + log.debug("Proxy candidate '{}' matches proxy source '{}': {}", + proxyCandidate.getHostName(), + fenceProxySource, + fromSelectedSource); + return fromSelectedSource; } - private boolean areAgentsVersionCompatible(VDS proxyCandidate) { - VdsFenceOptions options = new VdsFenceOptions(proxyCandidate.getVdsGroupCompatibilityVersion().getValue()); - boolean supported = true; - for (FenceAgent agent : proxyCandidate.getFenceAgents()) { - supported = supported && options.isAgentSupported(agent.getType()); + protected boolean areAgentsVersionCompatible(VDS proxyCandidate) { + VdsFenceOptions options = createVdsFenceOptions(proxyCandidate.getVdsGroupCompatibilityVersion().getValue()); + boolean compatible = true; + for (FenceAgent agent : fencedHost.getFenceAgents()) { + if (!options.isAgentSupported(agent.getType())) { + compatible = false; + break; + } } + + log.debug("Proxy candidate '{}' has compatible fence agents: {}", + proxyCandidate.getHostName(), + compatible); + return compatible; + } + + protected boolean isFencingPolicySupported(VDS proxyCandidate, Version minimalSupportedVersion) { + boolean supported = fencingPolicy == null + || proxyCandidate.getSupportedClusterVersionsSet().contains(minimalSupportedVersion); + + log.debug("Proxy candidate '{}' supports fencing policy '{}': {}", + proxyCandidate.getHostName(), + fencingPolicy, + supported); return supported; } - private boolean isFencingPolicySupported(VDS proxyCandidate, Version minimalSupportedVersion) { - return proxyCandidate.getSupportedClusterVersionsSet().contains(minimalSupportedVersion); - } - - private boolean isHostNetworkUnreachable(VDS proxyCandidate) { - return proxyCandidate.getStatus() == VDSStatus.Down + protected boolean isHostNetworkUnreachable(VDS proxyCandidate) { + boolean unreachable = proxyCandidate.getStatus() == VDSStatus.Down || proxyCandidate.getStatus() == VDSStatus.Reboot || proxyCandidate.getStatus() == VDSStatus.Kdumping || proxyCandidate.getStatus() == VDSStatus.NonResponsive || proxyCandidate.getStatus() == VDSStatus.PendingApproval || (proxyCandidate.getStatus() == VDSStatus.NonOperational && proxyCandidate.getNonOperationalReason() == NonOperationalReason.NETWORK_UNREACHABLE); + + log.debug("Proxy candidate '{}' with status '{}' is unreachable: {}", + proxyCandidate.getHostName(), + proxyCandidate.getStatus(), + unreachable); + return unreachable; } - public FencingPolicy getFencingPolicy() { - return fencingPolicy; + protected List<FenceProxySourceType> getDefaultFenceProxySources() { + return FenceProxySourceTypeHelper.parseFromString( + Config.<String>getValue(ConfigValues.FenceProxyDefaultPreferences)); } - public void setFencingPolicy(FencingPolicy fencingPolicy) { - this.fencingPolicy = fencingPolicy; + protected int getFindFenceProxyRetries() { + // make sure that loop is executed at least once , no matter what is the value in config + return Math.max(Config.<Integer>getValue(ConfigValues.FindFenceProxyRetries), 1); + } + + protected long getDelayBetweenRetries() { + return TimeUnit.SECONDS.toMillis( + Config.<Integer>getValue(ConfigValues.FindFenceProxyDelayBetweenRetriesInSec)); + } + + protected Version getMinSupportedVersionForFencingPolicy() { + return fencingPolicy == null ? null : FencingPolicyHelper.getMinimalSupportedVersion(fencingPolicy); + } + + protected VdsFenceOptions createVdsFenceOptions(String version) { + return new VdsFenceOptions(version); + } + + // TODO Investigate if injection is possible + protected DbFacade getDbFacade() { + return DbFacade.getInstance(); } } diff --git a/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/FenceProxyLocatorTest.java b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/FenceProxyLocatorTest.java index ca08b46..aa02b86 100644 --- a/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/FenceProxyLocatorTest.java +++ b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/pm/FenceProxyLocatorTest.java @@ -1,30 +1,39 @@ package org.ovirt.engine.core.bll.pm; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; +import org.junit.After; import org.junit.Before; -import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; import org.ovirt.engine.core.bll.DbDependentTestBase; +import org.ovirt.engine.core.common.businessentities.FenceAgent; import org.ovirt.engine.core.common.businessentities.FencingPolicy; import org.ovirt.engine.core.common.businessentities.NonOperationalReason; import org.ovirt.engine.core.common.businessentities.VDS; import org.ovirt.engine.core.common.businessentities.VDSStatus; -import org.ovirt.engine.core.common.config.ConfigValues; +import org.ovirt.engine.core.common.businessentities.pm.FenceProxySourceType; import org.ovirt.engine.core.compat.Guid; import org.ovirt.engine.core.compat.Version; import org.ovirt.engine.core.dal.dbbroker.DbFacade; import org.ovirt.engine.core.dao.VdsDAO; -import org.ovirt.engine.core.utils.MockConfigRule; +import org.ovirt.engine.core.utils.pm.VdsFenceOptions; /** * This class tests the FenceProxyLocator @@ -32,292 +41,353 @@ @RunWith(MockitoJUnitRunner.class) public class FenceProxyLocatorTest extends DbDependentTestBase { - private static String HOST_NAME = "hostname"; - private static String ANOTHER_HOST_NAME = "hostname2"; private static Guid FENCECD_HOST_ID = new Guid("11111111-1111-1111-1111-111111111111"); private static Guid FENCED_HOST_CLUSTER_ID = new Guid("22222222-2222-2222-2222-222222222222"); private static Guid FENCED_HOST_DATACENTER_ID = new Guid("33333333-3333-3333-3333-333333333333"); - private static Guid OTHER_HOST_ID_1 = new Guid("55555555-5555-5555-5555-555555555555"); - private static Guid OTHER_HOST_ID_2 = Guid.Empty; private static Guid OTHER_CLUSTER_ID = new Guid("66666666-6666-6666-6666-666666666666"); private static Guid OTHER_CLUSTER_ID_2 = new Guid("88888888-8888-8888-8888-888888888888"); private static Guid OTHER_DATACENTER_ID = new Guid("77777777-7777-7777-7777-777777777777"); - @ClassRule - public static MockConfigRule configRule = - new MockConfigRule(MockConfigRule.mockConfig(ConfigValues.FindFenceProxyRetries, 2), - MockConfigRule.mockConfig(ConfigValues.FindFenceProxyDelayBetweenRetriesInSec, 2), - MockConfigRule.mockConfig(ConfigValues.FenceProxyDefaultPreferences, "cluster,dc,other_dc"), - MockConfigRule.mockConfig(ConfigValues.VdsFenceOptionTypes, "secure=bool,port=int,slot=int")); - @Mock - private VDS fencedVds; - private DbFacade dbFacade; @Mock private VdsDAO vdsDao; - private FenceProxyLocator fenceProxyLocator; + private VdsFenceOptions vdsFenceOptions; + + private VDS fencedHost; @Before public void setup() { - dbFacade = DbFacade.getInstance(); - when(fencedVds.getName()).thenReturn(HOST_NAME); - when(fencedVds.getId()).thenReturn(FENCECD_HOST_ID); - when(fencedVds.getVdsGroupId()).thenReturn(FENCED_HOST_CLUSTER_ID); - when(fencedVds.getStoragePoolId()).thenReturn(FENCED_HOST_DATACENTER_ID); - fenceProxyLocator = new FenceProxyLocator(fencedVds); - when(dbFacade.getVdsDao()).thenReturn(vdsDao); + doReturn(vdsDao).when(dbFacade).getVdsDao(); + + mockVdsFenceOptions(true); + mockFencedHost(); + } + + @After + public void cleanup() { + fencedHost = null; } /** - * Tests flow where a proxy host is succesfully found + * Checks if locator select the only available host as a proxy. */ @Test public void findProxyHost() { - List<VDS> hosts = new LinkedList<>(); - VDS vds = createProxyCandidate(); - hosts.add(vds); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(); + mockExistingHosts(createHost()); + + VDS proxyHost = setupLocator().findProxyHost(); + assertNotNull(proxyHost); - assertEquals(proxyHost.getId(), OTHER_HOST_ID_1); } /** - * Tests that the locator doesn't choose the fenced-host as a proxy host. Validates that if this host is the only - * host in the list of proxy candidates, the method returns null. + * Checks if the locator excludes fenced host as a proxy host. And because fenced host is the only existing host, + * no proxy is selected */ @Test - public void findProxyHostExcludeSelf() { - List<VDS> hosts = new LinkedList<>(); - hosts.add(fencedVds); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(); + public void findProxyHostExcludesFencedHost() { + mockExistingHosts(fencedHost); + + VDS proxyHost = setupLocator().findProxyHost(); + assertNull(proxyHost); } - /** - * Tests that the locator doesn't choose the fenced-host as a proxy host. Validates that if there are additional - * hosts in the list of proxy candidates, the fence-host is skipped and another host is chosen. + * Checks if the locator excludes fenced host as a proxy host and selects different available host. */ @Test - public void findProxyHostExcludeSelfAnotherHostAvailable() { - List<VDS> hosts = new LinkedList<>(); - hosts.add(fencedVds); - hosts.add(createProxyCandidate()); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(); + public void findProxyHostExcludeFencedHostWhenOtherHostAvailable() { + mockExistingHosts(fencedHost, createHost()); + + VDS proxyHost = setupLocator().findProxyHost(); + assertNotNull(proxyHost); - assertEquals(proxyHost.getId(), OTHER_HOST_ID_1); + assertNotEquals(proxyHost.getId(), fencedHost.getId()); } - /** - * Tests that when a certain host is provided as one which should not be selected as proxy, it is indeed skipped. - * This test validates that if this host is the only host in the list of proxy candidates, the method returns null. + * Checks if the locator excludes specified host as a proxy host. And because specified host is the only existing + * host, no proxy is selected */ @Test - public void findProxyHostExcludeHost() { - List<VDS> hosts = new LinkedList<>(); - hosts.add(createProxyCandidate()); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(false, OTHER_HOST_ID_1); + public void findProxyHostExcludesSpecifiedHost() { + VDS excludedHost = createHost(); + mockExistingHosts(excludedHost); + + VDS proxyHost = setupLocator().findProxyHost(false, excludedHost.getId()); + assertNull(proxyHost); } - /** - * Tests that when a certain host is provided as one which should not be selected as proxy, it is indeed skipped. - * This test validates that if there are other hosts in the list of proxy candidates, one of them will be chosen, - * and not the host that should be skipped. + * Checks if the locator excludes specified host as a proxy host and select different available host. */ @Test public void findProxyHostExcludeHostAnotherHostAvailable() { - List<VDS> hosts = new LinkedList<>(); - hosts.add(createProxyCandidate()); - VDS anotherHost = createProxyCandidate(); - anotherHost.setId(OTHER_HOST_ID_2); - hosts.add(anotherHost); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(false, OTHER_HOST_ID_1); + VDS excludedHost = createHost(); + mockExistingHosts(excludedHost, createHost()); + + VDS proxyHost = setupLocator().findProxyHost(false, excludedHost.getId()); + assertNotNull(proxyHost); - assertEquals(proxyHost.getId(), OTHER_HOST_ID_2); + assertNotEquals(proxyHost.getId(), excludedHost.getId()); } - /** - * Tests that an unreachable host (a host in 'NonResponsive' state) isn't chosen as proxy. This test validates that - * if such a host is the only host in the list of proxy candidates, the method returns null. + * Checks if the locator excludes specified host as a proxy host, because it doesn't contain fence agents compatible + * with agents defined for fenced host. And because specified host is the only existing host, no proxy is selected */ @Test - public void findProxyHostExcludeUnreachable() { - List<VDS> hosts = new LinkedList<>(); - VDS vds = new VDS(); - vds.setId(OTHER_HOST_ID_1); - vds.setStatus(VDSStatus.NonResponsive); - vds.setVdsGroupId(FENCED_HOST_CLUSTER_ID); - vds.setVdsGroupCompatibilityVersion(Version.v3_5); - vds.getDynamicData().setNonOperationalReason(NonOperationalReason.NETWORK_UNREACHABLE); - hosts.add(vds); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(false); + public void findProxyHostExcludesHostWithIncompatibleAgents() { + VDS host = createHost(); + mockExistingHosts(host); + mockVdsFenceOptions(false); + + VDS proxyHost = setupLocator().findProxyHost(false); + assertNull(proxyHost); } - /** - * Tests that an unreachable host (a host in 'NonResponsive' state) isn't chosen as proxy. This test validates that - * if such there are additional hosts in the list of proxy candidates, one of them will be chosen, and not the - * unreachable host. + * Checks if the locator excludes specified host as a proxy host, because its supported cluster level is lower + * than minimal supported cluster level required by fencing policy. And because specified host is the only + * existing host, no proxy is selected */ @Test - public void findProxyHostExcludeUnreachableAnotherHostAvailable() { - List<VDS> hosts = new LinkedList<>(); - VDS vds = new VDS(); - vds.setId(OTHER_HOST_ID_2); - vds.setStatus(VDSStatus.NonResponsive); - vds.setVdsGroupId(FENCED_HOST_CLUSTER_ID); - vds.setVdsGroupCompatibilityVersion(Version.v3_5); - vds.getDynamicData().setNonOperationalReason(NonOperationalReason.NETWORK_UNREACHABLE); - hosts.add(vds); - hosts.add(createProxyCandidate()); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(false); - assertNotNull(proxyHost); - assertEquals(proxyHost.getId(), OTHER_HOST_ID_1); - } + public void findProxyHostExcludesHostDueToFencingPolicy() { + mockExistingHosts(createHost()); + FenceProxyLocator locator = setupLocator(new FencingPolicy()); + setMinSupportedVersionForFencingPolicy(locator, Version.v3_5); + VDS proxyHost = locator.findProxyHost(false); - /** - * Tests that the proxy locator will prefer a host from the same cluster as the fenced-host over a host from a - * different cluster (the test assumes: FenceProxyDefaultPreferences="cluster,dc,other_dc") - */ - @Test - public void preferProxyHostFromSameCluster() { - List<VDS> hosts = new LinkedList<>(); - VDS vds = new VDS(); - vds.setId(OTHER_HOST_ID_1); - vds.setVdsGroupId(OTHER_CLUSTER_ID); - vds.setVdsGroupCompatibilityVersion(Version.v3_5); - vds = new VDS(); - vds.setId(OTHER_HOST_ID_2); - vds.setVdsGroupId(FENCED_HOST_CLUSTER_ID); - vds.setVdsGroupCompatibilityVersion(Version.v3_5); - hosts.add(vds); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(false); - assertNotNull(proxyHost); - assertEquals(proxyHost.getId(), OTHER_HOST_ID_2); + assertNull(proxyHost); } /** - * Tests that, in a situation where there is no host availabe from the same cluster, the locator will prefer a host - * from the same datacenter as the fenced-host over a host from a different datacenter (the test assumes: - * FenceProxyDefaultPreferences="cluster,dc,other_dc") + * Checks if the locator excludes specified host as a proxy host, because it's unreachable by network. And because + * specified host is the only existing host, no proxy is selected. + * + * Special case when host has NonOperational status and reason NETWORK_UNREACHABLE is tested in different test case! */ @Test - public void preferProxyHostFromSameDatacenter() { - when(vdsDao.getAll()).thenReturn(createHosts()).thenReturn(createHosts()); - VDS proxyHost = fenceProxyLocator.findProxyHost(false); - assertNotNull(proxyHost); - assertEquals(proxyHost.getId(), OTHER_HOST_ID_2); + public void findProxyHostExcludesUnreachableHosts() { + FenceProxyLocator locator = setupLocator(); + + for (VDSStatus status : VDSStatus.values()) { + mockExistingHosts(createHost(status)); + + assertEquals(shouldHostBeUnreachable(status), locator.findProxyHost(false) == null); + } } + /** + * Checks if the locator excludes NonOperational host as a proxy host, if it's NonOperationalReason is + * NETWORK_UNREACHABLE. And because specified host is the only existing host, no proxy is selected. + */ + @Test + public void findProxyHostExcludesNonOperationalHosts() { + mockExistingHosts(createHost()); + FenceProxyLocator locator = setupLocator(); + + for (NonOperationalReason reason : NonOperationalReason.values()) { + VDS host = createHost(); + host.setStatus(VDSStatus.NonOperational); + host.setNonOperationalReason(reason); + mockExistingHosts(host); + + assertEquals(reason == NonOperationalReason.NETWORK_UNREACHABLE, locator.findProxyHost(false) == null); + } + } /** - * Tests that the locator will prefer an 'UP' host as proxy over a host in 'Maintenance' mode. + * Checks if the locator will prefer an host in Up status as proxy over a host in Maintenance status. */ @Test public void findProxyHostPreferUpHost() { - List<VDS> hosts = new LinkedList<>(); - VDS vds = new VDS(); - vds.setId(OTHER_HOST_ID_1); - vds.setVdsGroupId(FENCED_HOST_CLUSTER_ID); - vds.setVdsGroupCompatibilityVersion(Version.v3_5); - vds.setStatus(VDSStatus.Maintenance); - hosts.add(vds); - vds = new VDS(); - vds.setId(OTHER_HOST_ID_2); - vds.setVdsGroupId(FENCED_HOST_CLUSTER_ID); - vds.setVdsGroupCompatibilityVersion(Version.v3_5); - vds.setStatus(VDSStatus.Up); - hosts.add(vds); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(false); + VDS hostInMaintenance = createHost(); + hostInMaintenance.setStatus(VDSStatus.Maintenance); + mockExistingHosts(hostInMaintenance, createHost()); + + VDS proxyHost = setupLocator().findProxyHost(false); + assertNotNull(proxyHost); - assertEquals(proxyHost.getId(), OTHER_HOST_ID_2); + assertNotEquals(proxyHost.getId(), hostInMaintenance.getId()); } - /** - * Tests version compatibility when a fencing-policy is defined. In this test, the validation is successful because - * the minimal version supporting fencing-policy exists in the hosts 'supportedClusterLevels' list. + * Checks if the locator will select as proxy host in 'Maintenance' */ @Test - public void findProxyHostFencingPolicySupported() { - FencingPolicy policy = new FencingPolicy(); - fenceProxyLocator.setFencingPolicy(policy); - VDS vds = new VDS(); - vds.setSupportedClusterLevels(Version.v3_0.toString()); - vds.setId(OTHER_HOST_ID_1); - vds.setVdsGroupId(FENCED_HOST_CLUSTER_ID); - vds.setVdsGroupCompatibilityVersion(Version.v3_5); - List<VDS> hosts = new LinkedList<>(); - hosts.add(vds); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(false); + public void findProxyHostSelectSomeHostIfNoneUp() { + VDS hostInMaintenance1 = createHost(); + hostInMaintenance1.setStatus(VDSStatus.Maintenance); + VDS hostInMaintenance2 = createHost(); + hostInMaintenance2.setStatus(VDSStatus.Maintenance); + mockExistingHosts(hostInMaintenance1, hostInMaintenance2); + + VDS proxyHost = setupLocator().findProxyHost(false); + assertNotNull(proxyHost); } - /** - * Tests version compatibility when a fencing-policy is defined. In this test, the validation fails because the - * minimal version supporting fencing-policy does not exist in the hosts 'supportedClusterLevels' list. + * Checks that the proxy locator will prefer a host from the same cluster as the fenced-host over a host from a + * different cluster (assuming default fence proxy sources: "cluster,dc") */ @Test - public void findProxyHostFencingPolicyNotSupported() { - FencingPolicy policy = new FencingPolicy(); - fenceProxyLocator.setFencingPolicy(policy); - VDS vds = new VDS(); - vds.setSupportedClusterLevels(Version.v3_1.toString()); - vds.setId(OTHER_HOST_ID_1); - vds.setVdsGroupId(FENCED_HOST_CLUSTER_ID); - vds.setVdsGroupCompatibilityVersion(Version.v3_5); - List<VDS> hosts = new LinkedList<>(); - hosts.add(vds); - when(vdsDao.getAll()).thenReturn(hosts); - VDS proxyHost = fenceProxyLocator.findProxyHost(false); - assertNull(proxyHost); + public void preferProxyHostFromSameCluster() { + mockExistingHosts( + createHost(VDSStatus.Up, OTHER_CLUSTER_ID, FENCED_HOST_DATACENTER_ID), + createHost(VDSStatus.Up, FENCED_HOST_CLUSTER_ID, FENCED_HOST_DATACENTER_ID)); + + VDS proxyHost = setupLocator().findProxyHost(false); + + assertNotNull(proxyHost); + assertEquals(proxyHost.getVdsGroupId(), FENCED_HOST_CLUSTER_ID); } + /** + * Checks that, in a situation where there is no host available from the same cluster, the locator will select a host + * from the same data center as the fenced host over a host from a different data center (assuming default fence + * proxy sources: "cluster,dc") + */ + @Test + public void preferProxyHostFromSameDC() { + mockExistingHosts( + createHost(VDSStatus.Up, OTHER_CLUSTER_ID_2, OTHER_DATACENTER_ID), + createHost(VDSStatus.Up, OTHER_CLUSTER_ID, FENCED_HOST_DATACENTER_ID)); - private List<VDS> createHosts() { - List<VDS> hosts = new LinkedList<>(); - VDS vds = new VDS(); - vds.setId(OTHER_HOST_ID_1); - vds.setVdsGroupId(OTHER_CLUSTER_ID); - vds.setStoragePoolId(OTHER_DATACENTER_ID); - vds.setVdsGroupCompatibilityVersion(Version.v3_5); - hosts.add(vds); - vds = new VDS(); - vds.setId(OTHER_HOST_ID_2); - vds.setVdsGroupId(OTHER_CLUSTER_ID_2); - vds.setStoragePoolId(FENCED_HOST_DATACENTER_ID); - vds.setVdsGroupCompatibilityVersion(Version.v3_5); - hosts.add(vds); - return hosts; + VDS proxyHost = setupLocator().findProxyHost(false); + + assertNotNull(proxyHost); + assertEquals(proxyHost.getStoragePoolId(), FENCED_HOST_DATACENTER_ID); } - private VDS createProxyCandidate() { - VDS proxyCandidate = new VDS(); - proxyCandidate.setVdsName(ANOTHER_HOST_NAME); - proxyCandidate.setId(OTHER_HOST_ID_1); - proxyCandidate.setVdsGroupId(FENCED_HOST_CLUSTER_ID); - proxyCandidate.setStoragePoolId(FENCED_HOST_DATACENTER_ID); - proxyCandidate.setVdsGroupCompatibilityVersion(Version.v3_0); - return proxyCandidate; + /** + * Checks that if fence proxy sources are set as "other_dc,cluster,dc", then available host from other DC is selected + * as proxy although there's also an available host in the same cluster (BZ1131411) + */ + @Test + public void findProxyHostAccordingToOtherDcClusterDc() { + mockProxySourcesForFencedHost( + Arrays.asList( + FenceProxySourceType.OTHER_DC, + FenceProxySourceType.CLUSTER, + FenceProxySourceType.DC)); + mockExistingHosts( + createHost(VDSStatus.Up, FENCED_HOST_CLUSTER_ID, FENCED_HOST_DATACENTER_ID), + createHost(VDSStatus.Up, OTHER_CLUSTER_ID_2, OTHER_DATACENTER_ID)); + + VDS proxyHost = setupLocator().findProxyHost(false); + + assertNotNull(proxyHost); + assertEquals(proxyHost.getStoragePoolId(), OTHER_DATACENTER_ID); + } + + private void mockFencedHost() { + fencedHost = mock(VDS.class); + doReturn(FENCECD_HOST_ID).when(fencedHost).getId(); + doReturn(FENCED_HOST_CLUSTER_ID).when(fencedHost).getVdsGroupId(); + doReturn(FENCED_HOST_DATACENTER_ID).when(fencedHost).getStoragePoolId(); + doReturn("fencedHost").when(fencedHost).getHostName(); + doReturn(Arrays.asList(createFenceAgent(FENCECD_HOST_ID, "ipmilan"))) + .when(fencedHost).getFenceAgents(); + } + + private void mockProxySourcesForFencedHost(List<FenceProxySourceType> fenceProxySources) { + doReturn(fenceProxySources).when(fencedHost).getFenceProxySources(); + } + + private FenceAgent createFenceAgent(Guid hostId, String type) { + FenceAgent agent = new FenceAgent(); + agent.setId(Guid.newGuid()); + agent.setHostId(hostId); + agent.setType(type); + return agent; + } + + private FenceProxyLocator setupLocator() { + return setupLocator(null); + } + + private FenceProxyLocator setupLocator(FencingPolicy fencingPolicy) { + FenceProxyLocator fenceProxyLocator = spy(new FenceProxyLocator(fencedHost, fencingPolicy)); + doReturn(dbFacade).when(fenceProxyLocator).getDbFacade(); + doReturn(vdsFenceOptions).when(fenceProxyLocator).createVdsFenceOptions(any(String.class)); + doReturn(0L).when(fenceProxyLocator).getDelayBetweenRetries(); + doReturn(1).when(fenceProxyLocator).getFindFenceProxyRetries(); + doReturn(Arrays.asList(FenceProxySourceType.CLUSTER, FenceProxySourceType.DC)) + .when(fenceProxyLocator).getDefaultFenceProxySources(); + + mockVdsFenceOptions(true); + return fenceProxyLocator; + } + + private void setMinSupportedVersionForFencingPolicy(FenceProxyLocator locator, Version version) { + doReturn(version).when(locator).getMinSupportedVersionForFencingPolicy(); + } + + private VDS createHost() { + return createHost(VDSStatus.Up); + } + + private VDS createHost(VDSStatus status) { + return createHost(status, FENCED_HOST_CLUSTER_ID, FENCED_HOST_DATACENTER_ID); + } + + private VDS createHost(VDSStatus status, Guid clusterId, Guid dcId) { + VDS host = new VDS(); + host.setId(Guid.newGuid()); + host.setVdsGroupId(clusterId); + host.setStoragePoolId(dcId); + host.setVdsGroupCompatibilityVersion(Version.v3_0); + host.setStatus(status); + host.setHostName("host-" + host.getId()); + return host; + } + + private List<VDS> createHostList(VDS... hosts) { + List<VDS> hostList = new LinkedList<>(); + hostList.addAll(Arrays.asList(hosts)); + return hostList; + } + + private void mockExistingHosts(final VDS... hosts) { + // we need to recreate the list on each call, because the list is altered in FenceProxyLocator + when(vdsDao.getAll()).thenAnswer( + new Answer<List<VDS>>() { + @Override + public List<VDS> answer(InvocationOnMock invocationOnMock) throws Throwable { + return createHostList(hosts); + } + } + ); + } + + private void mockVdsFenceOptions(boolean agentsCompatibleWithProxy) { + vdsFenceOptions = mock(VdsFenceOptions.class); + doReturn(agentsCompatibleWithProxy).when(vdsFenceOptions).isAgentSupported(any(String.class)); + } + + private boolean shouldHostBeUnreachable(VDSStatus status) { + boolean unreachable; + switch(status) { + case Down: + case Reboot: + case Kdumping: + case NonResponsive: + case PendingApproval: + unreachable = true; + break; + + default: + unreachable = false; + } + return unreachable; } } -- To view, visit https://gerrit.ovirt.org/39764 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I59d5b46273dbaeb2d73142f1256000e36a6cbdd8 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Martin Peřina <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
