FGCP: catch exceptions when node is not in expected state when rebooting, deleting, etc. and ensure it reaches the final state the user wants
Project: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs/commit/d3a180ef Tree: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs/tree/d3a180ef Diff: http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs/diff/d3a180ef Branch: refs/heads/master Commit: d3a180eff84db4ac65a32d9b4d318322e4aa77f5 Parents: 2f11286 Author: Dies Koper <[email protected]> Authored: Wed May 22 15:22:58 2013 +1000 Committer: Andrew Gaul <[email protected]> Committed: Thu May 23 16:21:51 2013 -0700 ---------------------------------------------------------------------- .../strategy/FGCPComputeServiceAdapter.java | 105 +++++++++------ .../fgcp/handlers/FGCPServerErrorRetryHandler.java | 5 +- .../fgcp/handlers/ResponseNotSuccessHandler.java | 2 +- .../fgcp/services/ErrorResponseExpectTest.java | 37 +++++- .../IllegalState_ALREADY_STOPPED-response.xml | 5 + .../responses/IllegalState_RUNNING-response.xml | 5 + 6 files changed, 111 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs/blob/d3a180ef/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/compute/strategy/FGCPComputeServiceAdapter.java ---------------------------------------------------------------------- diff --git a/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/compute/strategy/FGCPComputeServiceAdapter.java b/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/compute/strategy/FGCPComputeServiceAdapter.java index 72b8975..e1cbfce 100644 --- a/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/compute/strategy/FGCPComputeServiceAdapter.java +++ b/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/compute/strategy/FGCPComputeServiceAdapter.java @@ -41,16 +41,17 @@ import org.jclouds.fujitsu.fgcp.compute.functions.ResourceIdToFirewallId; import org.jclouds.fujitsu.fgcp.compute.functions.ResourceIdToSystemId; import org.jclouds.fujitsu.fgcp.compute.predicates.ServerStarted; import org.jclouds.fujitsu.fgcp.compute.predicates.ServerStopped; -import org.jclouds.fujitsu.fgcp.compute.predicates.SystemStatusNormal; import org.jclouds.fujitsu.fgcp.compute.strategy.VServerMetadata.Builder; import org.jclouds.fujitsu.fgcp.domain.DiskImage; import org.jclouds.fujitsu.fgcp.domain.ServerType; +import org.jclouds.fujitsu.fgcp.domain.VServer; import org.jclouds.fujitsu.fgcp.domain.VServerStatus; import org.jclouds.fujitsu.fgcp.domain.VServerWithDetails; import org.jclouds.fujitsu.fgcp.domain.VServerWithVNICs; import org.jclouds.fujitsu.fgcp.domain.VSystem; import org.jclouds.fujitsu.fgcp.domain.VSystemWithDetails; import org.jclouds.logging.Logger; +import org.jclouds.rest.ResourceNotFoundException; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableSet; @@ -73,20 +74,18 @@ public class FGCPComputeServiceAdapter implements protected Predicate<String> serverStopped = null; protected Predicate<String> serverStarted = null; protected Predicate<String> serverCreated = null; - protected Predicate<String> systemNormal = null; protected ResourceIdToFirewallId toFirewallId = null; protected ResourceIdToSystemId toSystemId = null; @Inject public FGCPComputeServiceAdapter(FGCPApi api, ServerStopped serverStopped, - ServerStarted serverStarted, SystemStatusNormal systemNormal, - Timeouts timeouts, ResourceIdToFirewallId toFirewallId, + ServerStarted serverStarted, Timeouts timeouts, + ResourceIdToFirewallId toFirewallId, ResourceIdToSystemId toSystemId) { this.api = checkNotNull(api, "api"); this.serverStopped = retry(checkNotNull(serverStopped), timeouts.nodeSuspended); this.serverStarted = retry(checkNotNull(serverStarted), timeouts.nodeRunning); this.serverCreated = retry(checkNotNull(serverStopped), timeouts.nodeRunning); - this.systemNormal = retry(checkNotNull(systemNormal), timeouts.nodeTerminated); this.toFirewallId = checkNotNull(toFirewallId, "ResourceIdToFirewallId"); this.toSystemId = checkNotNull(toSystemId, "ResourceIdToSystemId"); } @@ -104,8 +103,7 @@ public class FGCPComputeServiceAdapter implements // wait until fully created (i.e. transitions to stopped status) checkState(serverCreated.apply(id), "node %s not reaching STOPPED state after creation", id); resumeNode(id); - // wait until fully started - checkState(serverStarted.apply(id), "could not start %s after creation", id); + // don't wait until fully started, template "optionToNotBlock" takes care of that VServerMetadata server = getNode(id); // do we need this? @@ -158,31 +156,34 @@ public class FGCPComputeServiceAdapter implements Builder builder = VServerMetadata.builder(); builder.id(id); - // mapped public ips? -// String fwId = toFirewallId.apply(id); - // futures.add(asyncApi.getBuiltinServerApi().getConfiguration(fwId, - // BuiltinServerConfiguration.SLB_RULE)); - - VServerWithDetails server = api.getVirtualServerApi().getDetails(id); - VServerStatus status = api.getVirtualServerApi().getStatus(id); - logger.info("%s [%s] - %s", id, status, server); - if (server == null) { - server = api.getVirtualServerApi().getDetails(id); - logger.warn("!!!!!!!!!! server returned null. 2nd try: %s", server); + try { + VServerWithDetails server = api.getVirtualServerApi().getDetails(id); + // skip FWs and SLBs + if (isFWorSLB(server)) { + return null; + } + VServerStatus status = api.getVirtualServerApi().getStatus(id); + logger.trace("Node %s [%s] - %s", id, status, server); + builder.serverWithDetails(server); + builder.status(status); + builder.initialPassword(api.getVirtualServerApi().getInitialPassword(id)); + + // mapped public ips? +// String fwId = toFirewallId.apply(id); + // futures.add(asyncApi.getBuiltinServerApi().getConfiguration(fwId, + // BuiltinServerConfiguration.FW_RULE)); + } catch (ResourceNotFoundException e) { + return null; } - builder.serverWithDetails(server); - builder.status(status == null ? VServerStatus.UNRECOGNIZED : status); - // System.out.println("status in adapter#getNode: " - // + (VServerStatus) results.get(1) - // +" for " - // + server.getId()); - builder.initialPassword(api.getVirtualServerApi().getInitialPassword(id)); - // SLB slb = ((BuiltinServer) results.get(4)).; - // slb. return builder.build(); } + private boolean isFWorSLB(VServer server) { + String serverType = server.getType(); + return "firewall".equals(serverType) || "slb".equals(serverType); + } + /** * {@inheritDoc} */ @@ -197,15 +198,9 @@ public class FGCPComputeServiceAdapter implements for (VServerWithVNICs server : systemDetails.getServers()) { - // skip FW (S-0001) and SLBs (>0 for SLB) - // TODO: rewrite to use serverType instead - if (!server.getId().endsWith("-S-0001") && server.getVnics().iterator().next().getNicNo() == 0) { - + // skip FWs and SLBs + if (!isFWorSLB(server)) { servers.add(getNode(server.getId())); - // Builder builder = VServerMetadata.builder(); - // builder.server(server); - // builder.status(VServerStatus.UNRECOGNIZED); - // servers.add(builder.build()); } } } @@ -233,9 +228,6 @@ public class FGCPComputeServiceAdapter implements suspendNode(id); checkState(serverStopped.apply(id), "could not stop %s before destroying it", id); api.getVirtualServerApi().destroy(id); - // wait until vsys completes reconfiguration - String systemId = toSystemId.apply(id); - checkState(systemNormal.apply(systemId), "system %s not returning to NORMAL", systemId); } /** @@ -253,7 +245,13 @@ public class FGCPComputeServiceAdapter implements */ @Override public void resumeNode(String id) { - api.getVirtualServerApi().start(id); + try { + api.getVirtualServerApi().start(id); + } catch (IllegalStateException ise) { + if (!(ise.getMessage().contains("ALREADY_STARTED") || ise.getMessage().contains("STARTING"))) { + throw ise; + } + } } /** @@ -261,6 +259,33 @@ public class FGCPComputeServiceAdapter implements */ @Override public void suspendNode(String id) { - api.getVirtualServerApi().stop(id); + try { + api.getVirtualServerApi().stop(id); + } catch (IllegalStateException ise) { + if (ise.getMessage().contains("ALREADY_STOPPED") || ise.getMessage().contains("STOPPING")) { + logger.trace("suspendNode({0}) - {1}", id, ise.getMessage()); + // ignore as it has/will reach the desired destination state + } else if (ise.getMessage().contains("STARTING")) { + // wait till running, then try to stop again + logger.trace("suspendNode({0}) - {1} - waiting to reach RUNNING state", id, ise.getMessage()); + checkState(serverStarted.apply(id), "starting %s didn't reach RUNNING state", id); + logger.trace("suspendNode({0}) - now RUNNING, trying to stop again", id); + try { + api.getVirtualServerApi().stop(id); + } catch (IllegalStateException e) { + if (e.getMessage().contains("ALREADY_STOPPED") || e.getMessage().contains("STOPPING")) { + logger.trace("suspendNode({0}) - {1}", id, e.getMessage()); + // ignore as it has/will reach the desired destination state + } else { + throw e; + } + } + } else { + throw ise; + } + } catch (RuntimeException e) { + logger.error(e, "suspendNode({0}) - exception occurred!", id); + throw e; + } } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs/blob/d3a180ef/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/handlers/FGCPServerErrorRetryHandler.java ---------------------------------------------------------------------- diff --git a/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/handlers/FGCPServerErrorRetryHandler.java b/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/handlers/FGCPServerErrorRetryHandler.java index 8c4da29..47936ca 100644 --- a/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/handlers/FGCPServerErrorRetryHandler.java +++ b/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/handlers/FGCPServerErrorRetryHandler.java @@ -25,7 +25,6 @@ import org.jclouds.http.HttpCommand; import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpRetryHandler; import org.jclouds.http.HttpUtils; -import org.jclouds.http.handlers.BackoffLimitedRetryHandler; import org.jclouds.logging.Logger; import com.google.inject.Singleton; @@ -37,7 +36,7 @@ import com.google.inject.Singleton; */ @Singleton public class FGCPServerErrorRetryHandler implements HttpRetryHandler { - private final BackoffLimitedRetryHandler backoffHandler; + private final HttpRetryHandler backoffHandler; @Inject @Named(Constants.PROPERTY_MAX_RETRIES) @@ -46,7 +45,7 @@ public class FGCPServerErrorRetryHandler implements HttpRetryHandler { protected Logger logger = Logger.NULL; @Inject - public FGCPServerErrorRetryHandler(BackoffLimitedRetryHandler backoffHandler) { + public FGCPServerErrorRetryHandler(FGCPBackoffLimitedRetryHandler backoffHandler) { this.backoffHandler = backoffHandler; } http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs/blob/d3a180ef/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/handlers/ResponseNotSuccessHandler.java ---------------------------------------------------------------------- diff --git a/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/handlers/ResponseNotSuccessHandler.java b/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/handlers/ResponseNotSuccessHandler.java index 9d4ea62..ec461dd 100644 --- a/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/handlers/ResponseNotSuccessHandler.java +++ b/fgcp/src/main/java/org/jclouds/fujitsu/fgcp/handlers/ResponseNotSuccessHandler.java @@ -48,7 +48,7 @@ public class ResponseNotSuccessHandler implements HttpErrorHandler { private static final Pattern RESOURCE_NOT_FOUND_PATTERN = Pattern.compile(".*RESOURCE_NOT_FOUND.*"); private static final Pattern NOT_IMPLEMENTED_PATTERN = Pattern.compile(".*NOTFOUND: API to the Version.*"); private static final Pattern ILLEGAL_STATE_PATTERN = Pattern - .compile(".*(NEVER_BOOTED|ALREADY_STARTED|ILLEGAL_STATE).*"); + .compile(".*(NEVER_BOOTED|ALREADY_STARTED|ALREADY_STOPPED|ILLEGAL_STATE).*"); private static final Pattern ILLEGAL_ARGUMENT_PATTERN = Pattern .compile(".*(SERVER_NAME_ALREADY_EXISTS|VALIDATION_ERROR).*"); http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs/blob/d3a180ef/fgcp/src/test/java/org/jclouds/fujitsu/fgcp/services/ErrorResponseExpectTest.java ---------------------------------------------------------------------- diff --git a/fgcp/src/test/java/org/jclouds/fujitsu/fgcp/services/ErrorResponseExpectTest.java b/fgcp/src/test/java/org/jclouds/fujitsu/fgcp/services/ErrorResponseExpectTest.java index 1c8f8b9..c651fc4 100644 --- a/fgcp/src/test/java/org/jclouds/fujitsu/fgcp/services/ErrorResponseExpectTest.java +++ b/fgcp/src/test/java/org/jclouds/fujitsu/fgcp/services/ErrorResponseExpectTest.java @@ -22,7 +22,6 @@ import static org.testng.Assert.fail; import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpResponse; -import org.jclouds.http.HttpResponseException; import org.jclouds.rest.AuthorizationException; import org.jclouds.rest.ResourceNotFoundException; import org.testng.annotations.Test; @@ -45,11 +44,11 @@ public class ErrorResponseExpectTest extends BaseFGCPRestApiExpectTest { api.createServer("vm1", "economy", "IMG_A1B2C3_1234567890ABCD", "ABCDEFGH-A123B456CE-N-DMZ"); fail("should have thrown an exception"); } catch (IllegalStateException e) { - assertTrue(e.getMessage().contains("RECONFIG_ING")); + assertTrue(e.getMessage().contains("RECONFIG_ING"), e.getMessage()); } } - public void testAuthenticationErrorDueToCertFromDifferentRegion() { + public void testAuthorizationErrorDueToCertFromDifferentRegion() { HttpRequest request = buildGETWithQuery("Action=CreateVServer" + "&vserverName=vm1" + "&vserverType=economy" + "&diskImageId=IMG_A1B2C3_1234567890ABCD" + "&networkId=ABCDEFGH-A123B456CE-N-DMZ" + "&vsysId=ABCDEFGH-A123B456CE"); @@ -65,6 +64,36 @@ public class ErrorResponseExpectTest extends BaseFGCPRestApiExpectTest { } } + public void testIllegalStateDueToStartingRunningServer() { + HttpRequest request = buildGETWithQuery("Action=StartVServer" + "&vserverId=ABCDEFGH-A123B456CE-S-0004" + + "&vsysId=ABCDEFGH-A123B456CE"); + HttpResponse response = HttpResponse.builder().statusCode(500) + .payload(payloadFromResource("/IllegalState_RUNNING-response.xml")).build(); + VirtualServerApi api = requestSendsResponse(request, response).getVirtualServerApi(); + + try { + api.start("ABCDEFGH-A123B456CE-S-0004"); + fail("should have thrown an exception"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("RUNNING"), e.getMessage()); + } + } + + public void testIllegalStateDueToStoppingStoppedServer() { + HttpRequest request = buildGETWithQuery("Action=StopVServer" + "&vserverId=ABCDEFGH-A123B456CE-S-0004" + + "&vsysId=ABCDEFGH-A123B456CE"); + HttpResponse response = HttpResponse.builder().statusCode(500) + .payload(payloadFromResource("/IllegalState_ALREADY_STOPPED-response.xml")).build(); + VirtualServerApi api = requestSendsResponse(request, response).getVirtualServerApi(); + + try { + api.stop("ABCDEFGH-A123B456CE-S-0004"); + fail("should have thrown an exception"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("ALREADY_STOPPED"), e.getMessage()); + } + } + public void testErrorDueToWrongResourceId() { HttpRequest request = buildGETWithQuery("Action=CreateVServer" + "&vserverName=vm1" + "&vserverType=economy" + "&diskImageId=IMG_DOES_NOT_EXIST" + "&networkId=ABCDEFGH-A123B456CE-N-DMZ" @@ -77,7 +106,7 @@ public class ErrorResponseExpectTest extends BaseFGCPRestApiExpectTest { api.createServer("vm1", "economy", "IMG_DOES_NOT_EXIST", "ABCDEFGH-A123B456CE-N-DMZ"); fail("should have thrown an exception"); } catch (ResourceNotFoundException e) { - assertTrue(e.getMessage().contains("RESOURCE_NOT_FOUND")); + assertTrue(e.getMessage().contains("RESOURCE_NOT_FOUND"), e.getMessage()); } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs/blob/d3a180ef/fgcp/src/test/resources/responses/IllegalState_ALREADY_STOPPED-response.xml ---------------------------------------------------------------------- diff --git a/fgcp/src/test/resources/responses/IllegalState_ALREADY_STOPPED-response.xml b/fgcp/src/test/resources/responses/IllegalState_ALREADY_STOPPED-response.xml new file mode 100644 index 0000000..37370a4 --- /dev/null +++ b/fgcp/src/test/resources/responses/IllegalState_ALREADY_STOPPED-response.xml @@ -0,0 +1,5 @@ +<?xml version="1.0" encoding="UTF-8"?> +<CreateVServerResponse xmlns="http://apioviss.jp.fujitsu.com";> + <responseMessage>The status of Instance[ABCDEFGH-A123B456CE-S-0004] is [STOPPED].</responseMessage> + <responseStatus>ALREADY_STOPPED</responseStatus> +</CreateVServerResponse> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-jclouds-labs/blob/d3a180ef/fgcp/src/test/resources/responses/IllegalState_RUNNING-response.xml ---------------------------------------------------------------------- diff --git a/fgcp/src/test/resources/responses/IllegalState_RUNNING-response.xml b/fgcp/src/test/resources/responses/IllegalState_RUNNING-response.xml new file mode 100644 index 0000000..b263e18 --- /dev/null +++ b/fgcp/src/test/resources/responses/IllegalState_RUNNING-response.xml @@ -0,0 +1,5 @@ +<?xml version="1.0" encoding="UTF-8"?> +<CreateVServerResponse xmlns="http://apioviss.jp.fujitsu.com";> + <responseMessage>The status of Instance[ABCDEFGH-A123B456CE-S-0004] is [RUNNING].</responseMessage> + <responseStatus>ALREADY_STARTED</responseStatus> +</CreateVServerResponse> \ No newline at end of file
