This is an automated email from the ASF dual-hosted git repository. heneveld pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/brooklyn-server.git
commit 88a5d6c91034e08e61cb35fe8ebb788bf99ad98f Author: Alex Heneveld <alex.henev...@cloudsoftcorp.com> AuthorDate: Fri Dec 4 09:15:06 2020 +0000 support for suspend and shutdown and getStatus and makeRunning on API and with jclouds (but shutdown actually just invokes suspend, because jclouds doesnt support "shutdown") --- .../api/location/MachineManagementMixins.java | 14 +- .../core/entity/trait/StartableMethods.java | 11 +- .../core/location/AbstractMachineLocation.java | 4 + .../core/location/BasicMachineMetadata.java | 25 +- .../core/location/MachineLifecycleUtils.java | 288 +++++++++++++++++++++ .../brooklyn/location/ssh/SshMachineLocation.java | 12 +- .../jclouds/DefaultConnectivityResolver.java | 5 +- .../brooklyn/location/jclouds/JcloudsLocation.java | 73 ++++-- .../location/jclouds/JcloudsMachineLocation.java | 10 +- .../jclouds/JcloudsSshMachineLocation.java | 12 +- ...cloudsLocationSuspendResumeMachineLiveTest.java | 46 +++- .../resources/brooklyn/logback-logger-excludes.xml | 4 + .../lifecycle/MachineLifecycleEffectorTasks.java | 6 +- 13 files changed, 458 insertions(+), 52 deletions(-) diff --git a/api/src/main/java/org/apache/brooklyn/api/location/MachineManagementMixins.java b/api/src/main/java/org/apache/brooklyn/api/location/MachineManagementMixins.java index cf6fa25..7043675 100644 --- a/api/src/main/java/org/apache/brooklyn/api/location/MachineManagementMixins.java +++ b/api/src/main/java/org/apache/brooklyn/api/location/MachineManagementMixins.java @@ -21,6 +21,7 @@ package org.apache.brooklyn.api.location; import java.util.Map; import com.google.common.annotations.Beta; +import org.apache.brooklyn.util.collections.MutableMap; /** * Defines mixins for interesting locations. @@ -83,7 +84,9 @@ public class MachineManagementMixins { @Beta public interface ResumesMachines { /** - * Resume the indicated machine. + * Resume the indicated machine. Map should have at minimum the `id` of the machine and + * the `user` and any other location-specific config keys for connecting subsequently. + * May return the original {@link MachineLocation} if found or may return a new {@link MachineLocation} if data might have changed. */ MachineLocation resumeMachine(Map<?, ?> flags); } @@ -97,19 +100,20 @@ public class MachineManagementMixins { void shutdownMachine(MachineLocation location); /** - * Ensure that a machine that might have been shutdown is running, or throw if not possible. - * May return the original {@link MachineLocation} or may return a new {@link MachineLocation} if data might have changed. + * Start up the indicated machine that might have been shutdown, or throw if not possible. + * May return the original {@link MachineLocation} if found or may return a new {@link MachineLocation} if data might have changed. */ - MachineLocation startupMachine(MachineLocation location); + MachineLocation startupMachine(Map<?, ?> flags); /** Issues a reboot command via the machine location provider (not on-box), or does a shutdown/startup pair * (but only if the implementation of {@link #shutdownMachine(MachineLocation)} does a true machine stop, not a suspend). */ - MachineLocation rebootMachine(MachineLocation location); + void rebootMachine(MachineLocation location); } @Beta public interface GivesMetrics { + /** * Gets metrics of a machine within a location. The actual metrics supported depends on the implementation, which should advise which config keys it supports. */ diff --git a/core/src/main/java/org/apache/brooklyn/core/entity/trait/StartableMethods.java b/core/src/main/java/org/apache/brooklyn/core/entity/trait/StartableMethods.java index 30b2348..933dd25 100644 --- a/core/src/main/java/org/apache/brooklyn/core/entity/trait/StartableMethods.java +++ b/core/src/main/java/org/apache/brooklyn/core/entity/trait/StartableMethods.java @@ -48,22 +48,23 @@ public class StartableMethods { /** Common implementation for start in parent nodes; just invokes start on all children of the entity */ public static void start(Entity e, Collection<? extends Location> locations) { - log.debug("Starting entity "+e+" at "+locations); + log.debug("Starting children of entity "+e+" at "+locations); DynamicTasks.get(startingChildren(e, locations), e); + log.debug("Started children of entity "+e); } /** Common implementation for stop in parent nodes; just invokes stop on all children of the entity */ public static void stop(Entity e) { - log.debug("Stopping entity "+e); + log.debug("Stopping children of entity "+e); DynamicTasks.get(stoppingChildren(e), e); - if (log.isDebugEnabled()) log.debug("Stopped entity "+e); + log.debug("Stopped children of entity "+e); } /** Common implementation for restart in parent nodes; just invokes restart on all children of the entity */ public static void restart(Entity e) { - log.debug("Restarting entity "+e); + log.debug("Restarting children of entity "+e); DynamicTasks.get(restartingChildren(e), e); - if (log.isDebugEnabled()) log.debug("Restarted entity "+e); + log.debug("Restarted children of entity "+e); } private static <T extends Entity> Iterable<T> filterStartableManagedEntities(Iterable<T> contenders) { diff --git a/core/src/main/java/org/apache/brooklyn/core/location/AbstractMachineLocation.java b/core/src/main/java/org/apache/brooklyn/core/location/AbstractMachineLocation.java index ae9e819..3775016 100644 --- a/core/src/main/java/org/apache/brooklyn/core/location/AbstractMachineLocation.java +++ b/core/src/main/java/org/apache/brooklyn/core/location/AbstractMachineLocation.java @@ -18,14 +18,18 @@ */ package org.apache.brooklyn.core.location; +import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Function; import org.apache.brooklyn.api.location.MachineDetails; import org.apache.brooklyn.api.location.MachineLocation; import org.apache.brooklyn.api.location.OsDetails; import org.apache.brooklyn.config.ConfigKey; import org.apache.brooklyn.core.config.ConfigKeys; import org.apache.brooklyn.core.entity.AbstractEntity; +import org.apache.brooklyn.util.collections.MutableList; import org.apache.brooklyn.util.collections.MutableMap; import org.apache.brooklyn.util.core.mutex.MutexSupport; import org.apache.brooklyn.util.core.mutex.WithMutexes; diff --git a/core/src/main/java/org/apache/brooklyn/core/location/BasicMachineMetadata.java b/core/src/main/java/org/apache/brooklyn/core/location/BasicMachineMetadata.java index fd0891f..0f7cef8 100644 --- a/core/src/main/java/org/apache/brooklyn/core/location/BasicMachineMetadata.java +++ b/core/src/main/java/org/apache/brooklyn/core/location/BasicMachineMetadata.java @@ -22,19 +22,33 @@ import org.apache.brooklyn.api.location.MachineManagementMixins; import com.google.common.base.MoreObjects; import com.google.common.base.Objects; +import org.apache.brooklyn.core.location.MachineLifecycleUtils.GivesMachineStatus; +import org.apache.brooklyn.core.location.MachineLifecycleUtils.MachineStatus; -public class BasicMachineMetadata implements MachineManagementMixins.MachineMetadata { +public class BasicMachineMetadata implements MachineManagementMixins.MachineMetadata, GivesMachineStatus { final String id, name, primaryIp; final Boolean isRunning; + final MachineStatus status; final Object originalMetadata; - + + public BasicMachineMetadata(String id, String name, String primaryIp, MachineStatus status, Object originalMetadata) { + super(); + this.id = id; + this.name = name; + this.primaryIp = primaryIp; + this.status = status; + this.isRunning = MachineStatus.RUNNING.equals(status); + this.originalMetadata = originalMetadata; + } + @Deprecated /** @deprecated since 1.1, use other constructor */ public BasicMachineMetadata(String id, String name, String primaryIp, Boolean isRunning, Object originalMetadata) { super(); this.id = id; this.name = name; this.primaryIp = primaryIp; this.isRunning = isRunning; + this.status = Boolean.TRUE.equals(isRunning) ? MachineStatus.RUNNING : MachineStatus.UNKNOWN; this.originalMetadata = originalMetadata; } @@ -59,6 +73,13 @@ public class BasicMachineMetadata implements MachineManagementMixins.MachineMeta } @Override + public MachineStatus getStatus() { + if (status!=null) return status; + if (isRunning()) return MachineStatus.RUNNING; + return MachineStatus.UNKNOWN; + } + + @Override public Object getOriginalMetadata() { return originalMetadata; } diff --git a/core/src/main/java/org/apache/brooklyn/core/location/MachineLifecycleUtils.java b/core/src/main/java/org/apache/brooklyn/core/location/MachineLifecycleUtils.java new file mode 100644 index 0000000..75c86a6 --- /dev/null +++ b/core/src/main/java/org/apache/brooklyn/core/location/MachineLifecycleUtils.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.location; + +import com.google.common.base.Preconditions; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.brooklyn.api.location.*; +import org.apache.brooklyn.api.location.MachineManagementMixins.GivesMachineMetadata; +import org.apache.brooklyn.api.location.MachineManagementMixins.GivesMetrics; +import org.apache.brooklyn.api.location.MachineManagementMixins.MachineMetadata; +import org.apache.brooklyn.config.ConfigKey; +import org.apache.brooklyn.core.config.ConfigKeys; +import org.apache.brooklyn.core.objs.BrooklynObjectInternal.ConfigurationSupportInternal; +import org.apache.brooklyn.util.JavaGroovyEquivalents; +import org.apache.brooklyn.util.collections.MutableList; +import org.apache.brooklyn.util.collections.MutableMap; +import org.apache.brooklyn.util.core.config.ConfigBag; +import org.apache.brooklyn.util.core.task.DynamicTasks; +import org.apache.brooklyn.util.core.task.Tasks; +import org.apache.brooklyn.util.exceptions.Exceptions; +import org.apache.brooklyn.util.exceptions.ReferenceWithError; +import org.apache.brooklyn.util.guava.Maybe; +import org.apache.brooklyn.util.time.CountdownTimer; +import org.apache.brooklyn.util.time.Duration; +import org.apache.brooklyn.util.time.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MachineLifecycleUtils { + + private static final Logger LOG = LoggerFactory.getLogger(MachineLifecycleUtils.class); + + private final MachineLocation location; + + public static ConfigKey<MachineStatus> STATUS = ConfigKeys.newConfigKey(MachineStatus.class, "status"); + + public MachineLifecycleUtils(MachineLocation l) { + this.location = l; + } + + public enum MachineStatus { + /** Either the machine is unknown at the location, or the machine may be known but status unknown or unrecognized, + * or we are unable to find out. Can use {@link #exists()} to determine the difference between these cases (respectively: false, true, null). */ + UNKNOWN, + RUNNING, + SHUTDOWN, + /** Note some providers report 'shutdown' as 'suspended' if they cannot tell the difference. */ + SUSPENDED, + TRANSITIONING, + ERROR + } + + public interface GivesMachineStatus { + MachineStatus getStatus(); + } + + /** true if the machine is known at its parent (at the actual provider), false if not known; null if cannot tell */ + @Nullable + public Boolean exists() { + if (location.getParent() instanceof MachineManagementMixins.GivesMachineMetadata) { + MachineMetadata metadata = ((GivesMachineMetadata) location.getParent()).getMachineMetadata(location); + return (metadata != null); + } + if (location.getParent() instanceof MachineManagementMixins.GivesMetrics) { + ConfigBag metrics = ConfigBag.newInstance(((GivesMetrics) location.getParent()).getMachineMetrics(location)); + return (!metrics.isEmpty()); + } + return null; + } + + @Nonnull + public MachineStatus getStatus() { + if (location.getParent() instanceof MachineManagementMixins.GivesMetrics) { + ConfigBag metrics = ConfigBag.newInstance(((GivesMetrics) location.getParent()).getMachineMetrics(location)); + MachineStatus s = metrics.get(STATUS); + if (s!=null) { + return s; + } + } + if (location.getParent() instanceof MachineManagementMixins.GivesMachineMetadata) { + MachineMetadata metadata = ((GivesMachineMetadata) location.getParent()).getMachineMetadata(location); + if (metadata!=null) { + if (metadata instanceof GivesMachineStatus) { + return ((GivesMachineStatus)metadata).getStatus(); + } + if (metadata.isRunning()) { + return MachineStatus.RUNNING; + } else { + return MachineStatus.UNKNOWN; + } + } + } + return MachineStatus.UNKNOWN; + } + + Duration timeout = Duration.minutes(30); + /** How long to delay on async operations, e.g. resume, including if machine was previuosly transitioning. Defaults 30 minutes. */ + public Duration getTimeout() { + return timeout; + } + public void setTimeout(Duration timeout) { + this.timeout = Preconditions.checkNotNull(timeout); + } + + /** + * Returns a masked error if the machine is already running. + * Unmasked error if the machine is not resumable and we cannot confirm it is already running. + * Otherwise returns the resumed machines status. + * <p> + * If in a transitional state will wait for the indicated timeout. + * <p> + * May return a different machine location than the one known here if critical metadata has changed (e.g. IP address); + * however the instance/ID will be the same. + */ + // TODO kill this? + public ReferenceWithError<MachineLocation> resume() { + MachineStatus status = getStatus(); + if (MachineStatus.RUNNING.equals(status)) return ReferenceWithError.newInstanceMaskingError(location, new Throwable("Already running")); + + if (location.getParent() instanceof MachineManagementMixins.ResumesMachines) { + try { + return ReferenceWithError.newInstanceWithoutError( ((MachineManagementMixins.ResumesMachines) location.getParent()).resumeMachine(getConfigMapWithId()) ); + } catch (Exception e) { + return ReferenceWithError.newInstanceThrowingError( location, e ); + } + } + + return ReferenceWithError.newInstanceThrowingError(location, new Throwable("Machine does not support resumption")); + } + + /** Returns supplied machine if already running; otherwise if suspended, resumes; if shutdown, starts it up and may return same or different object. + * If can't restart, or can't detect, it returns an absent. */ + public Maybe<MachineLocation> makeRunning() { + + CountdownTimer timer = CountdownTimer.newInstanceStarted(getTimeout()); + + MachineStatus status = getStatus(); + Duration sleep = Duration.ONE_SECOND; + while (MachineStatus.TRANSITIONING.equals(status)) { + if (timer.isExpired()) { + return Maybe.absent("Timeout waiting for "+location+" to be stable before running"); + } + try { + Duration s = sleep; + Tasks.withBlockingDetails("waiting on "+location+" to be stable before running", () -> { + Time.sleep(Duration.min(s, timer.getDurationRemaining())); + return null; + }); + } catch (Exception e) { + return Maybe.absent(new IllegalStateException("Error waiting for "+location+" to be stable before running", e)); + } + status = getStatus(); + sleep = Duration.min(sleep.multiply(1.2), Duration.ONE_MINUTE); + } + + if (MachineStatus.RUNNING.equals(status)) return Maybe.of(location); + + if (MachineStatus.SUSPENDED.equals(status) && location.getParent() instanceof MachineManagementMixins.ResumesMachines) { + return Maybe.of( ((MachineManagementMixins.ResumesMachines) location.getParent()).resumeMachine(getConfigMapWithId()) ); + } + if (MachineStatus.SHUTDOWN.equals(status) && location.getParent() instanceof MachineManagementMixins.ShutsdownMachines) { + return Maybe.of( ((MachineManagementMixins.ShutsdownMachines) location.getParent()).startupMachine(getConfigMapWithId()) ); + } + + return Maybe.absent("Unable to make "+location+" running from status "+status+"; no methods for doing so are available"); + } + + public MachineLocation makeRunningOrRecreate(Map<String,?> newConfig) throws NoMachinesAvailableException { + Exception problemRunning = null; + Maybe<MachineLocation> result = null; + try { + result = makeRunning(); + if (result.isPresent()) { + return result.get(); + } + problemRunning = Maybe.Absent.getException(result); + // couldn't resume etc + + } catch (Exception e) { + problemRunning = e; + } + + if (problemRunning!=null) { + LOG.warn("Unable to make existing machine running (" + location + "), will destroy and re-create: " + problemRunning); + if (LOG.isTraceEnabled()) { + LOG.trace("Trace for: Unable to make existing machine running (" + location + "), will destroy and re-create: " + problemRunning, problemRunning); + } + + DynamicTasks.queueIfPossible(Tasks.warning("Could not make existing machine running: " + Exceptions.collapseText(problemRunning), problemRunning)); + } + + if (!(location.getParent() instanceof MachineProvisioningLocation)) { + throw new IllegalStateException("Cannot destroy/recreate "+location+" because parent is not a provisioning location, and cannot resume due to: "+problemRunning); + + } + ((MachineProvisioningLocation)location.getParent()).release(location); + + return ((MachineProvisioningLocation)location.getParent()).obtain(newConfig); + } + + public ConfigBag getConfig() { + return ((ConfigurationSupportInternal)location.config()).getBag(); + } + + public Map<String,?> getConfigMapWithId() { + MutableMap<String, Object> result = MutableMap.copyOf(getConfig().getAllConfig()); + if (location.getParent() instanceof MachineManagementMixins.GivesMachineMetadata) { + MachineMetadata metadata = ((GivesMachineMetadata) location.getParent()).getMachineMetadata(location); + if (metadata!=null) { + result.add("id", metadata.getId()); + } + } + return result; + } + + /** If two locations point at the same instance; primarily looking at instance IDs, optionally also looking at addressibility. + * + * Locations are "primarily" the same if they have the same instance ID. + * (Would be nice to confirm that the parents are the same, but that's harder, and not necessary for our use cases.) + * <p> + * "Optionally" they can be addressible the same if they have the same public and private IPs and user access (maybe creds, depending on implementations). + * <p> + * Null may be returned if we cannot tell. + */ + public static Boolean isSameInstance(MachineLocation m1, MachineLocation m2, boolean requireSameAccessDetails) { + Boolean primarilySame = null; + Location p1 = m1.getParent(); + Location p2 = m2.getParent(); + if (p1 instanceof GivesMachineMetadata || p2 instanceof GivesMachineMetadata) { + if (p1 instanceof GivesMachineMetadata && p2 instanceof GivesMachineMetadata) { + String m1id = ((GivesMachineMetadata)p1).getMachineMetadata(m1).getId(); + String m2id = ((GivesMachineMetadata)p2).getMachineMetadata(m2).getId(); + if (Objects.equals(m1id, m2id)) { + if (m1id!=null) { + primarilySame = true; + } else { + // indeterminate + } + } else { + primarilySame = false; + } + } else { + return false; + } + } + + if (Boolean.FALSE.equals(primarilySame)) return false; + + if (requireSameAccessDetails) { + List<Function<MachineLocation,Object>> reqs = MutableList.of(); + reqs.add(MachineLocation::getUser); + reqs.add(MachineLocation::getHostname); + reqs.add(MachineLocation::getAddress); + reqs.add(MachineLocation::getPrivateAddresses); + reqs.add(MachineLocation::getPublicAddresses); + + for (Function<MachineLocation,Object> f: reqs) { + Object v1 = f.apply(m1); + if (!Objects.equals(v1, f.apply(m2))) return false; + if (JavaGroovyEquivalents.groovyTruth(v1)) primarilySame = true; + } + } + + return primarilySame; + } + +} diff --git a/core/src/main/java/org/apache/brooklyn/location/ssh/SshMachineLocation.java b/core/src/main/java/org/apache/brooklyn/location/ssh/SshMachineLocation.java index d5ed481..577e1d9 100644 --- a/core/src/main/java/org/apache/brooklyn/location/ssh/SshMachineLocation.java +++ b/core/src/main/java/org/apache/brooklyn/location/ssh/SshMachineLocation.java @@ -950,12 +950,22 @@ public class SshMachineLocation extends AbstractMachineLocation implements Machi } } + Duration sshCheckTimeout = null; + @Beta + public void setSshCheckTimeout(Duration sshCheckTimeout) { + this.sshCheckTimeout = sshCheckTimeout; + } + @Beta + public Duration getSshCheckTimeout() { + return Maybe.ofDisallowingNull(sshCheckTimeout).or(Duration.millis(SSHABLE_CONNECT_TIMEOUT)); + } + public boolean isSshable() { String cmd = "date"; try { try { Socket s = new Socket(); - s.connect(new InetSocketAddress(getAddress(), getPort()), SSHABLE_CONNECT_TIMEOUT); + s.connect(new InetSocketAddress(getAddress(), getPort()), (int) getSshCheckTimeout().toMilliseconds()); s.close(); } catch (IOException e) { if (LOG.isDebugEnabled()) LOG.debug(""+this+" not [yet] reachable (socket "+getAddress()+":"+getPort()+"): "+e); diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/DefaultConnectivityResolver.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/DefaultConnectivityResolver.java index ef02d2d..f5cf624 100644 --- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/DefaultConnectivityResolver.java +++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/DefaultConnectivityResolver.java @@ -31,6 +31,7 @@ import org.apache.brooklyn.api.entity.Entity; import org.apache.brooklyn.api.entity.EntityInitializer; import org.apache.brooklyn.api.entity.EntityLocal; import org.apache.brooklyn.config.ConfigKey; +import org.apache.brooklyn.core.BrooklynVersion; import org.apache.brooklyn.core.config.ConfigKeys; import org.apache.brooklyn.core.entity.Attributes; import org.apache.brooklyn.core.entity.BrooklynConfigKeys; @@ -172,6 +173,9 @@ public class DefaultConnectivityResolver extends InitializerPatternForConfigurab final Stopwatch timer = Stopwatch.createStarted(); // Should only be null in tests. final Entity contextEntity = getContextEntity(config); + if (contextEntity==null && !BrooklynVersion.isDevelopmentEnvironment()) { + LOG.debug("No context entity found in config or current task when resolving "+location); + } if (shouldPublishNetworks() && !options.isRebinding() && contextEntity != null) { publishNetworks(node, contextEntity); } @@ -468,7 +472,6 @@ public class DefaultConnectivityResolver extends InitializerPatternForConfigurab return taskContext; } } - LOG.warn("No context entity found in config or current task"); return null; } diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java index 516cbf1..b03575b 100644 --- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java +++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java @@ -20,7 +20,11 @@ package org.apache.brooklyn.location.jclouds; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import java.util.stream.Collectors; +import org.apache.brooklyn.api.location.*; import org.apache.brooklyn.api.location.MachineManagementMixins.MachineMetadata; +import org.apache.brooklyn.core.location.*; +import org.apache.brooklyn.core.location.MachineLifecycleUtils.MachineStatus; import static org.apache.brooklyn.util.JavaGroovyEquivalents.elvis; import static org.apache.brooklyn.util.JavaGroovyEquivalents.groovyTruth; import static org.apache.brooklyn.util.ssh.BashCommands.sbinPath; @@ -50,24 +54,13 @@ import javax.annotation.Nullable; import javax.xml.ws.WebServiceException; import org.apache.brooklyn.api.entity.Entity; -import org.apache.brooklyn.api.location.LocationSpec; -import org.apache.brooklyn.api.location.MachineLocation; -import org.apache.brooklyn.api.location.MachineLocationCustomizer; -import org.apache.brooklyn.api.location.MachineManagementMixins; -import org.apache.brooklyn.api.location.NoMachinesAvailableException; -import org.apache.brooklyn.api.location.PortRange; import org.apache.brooklyn.api.mgmt.AccessController; import org.apache.brooklyn.api.mgmt.Task; import org.apache.brooklyn.config.ConfigKey; import org.apache.brooklyn.config.ConfigKey.HasConfigKey; import org.apache.brooklyn.core.config.ConfigUtils; import org.apache.brooklyn.core.config.Sanitizer; -import org.apache.brooklyn.core.location.AbstractLocation; -import org.apache.brooklyn.core.location.BasicMachineMetadata; -import org.apache.brooklyn.core.location.LocationConfigKeys; -import org.apache.brooklyn.core.location.LocationConfigUtils; import org.apache.brooklyn.core.location.LocationConfigUtils.OsCredential; -import org.apache.brooklyn.core.location.PortRanges; import org.apache.brooklyn.core.location.access.PortForwardManager; import org.apache.brooklyn.core.location.access.PortForwardManagerLocationResolver; import org.apache.brooklyn.core.location.access.PortMapping; @@ -554,10 +547,25 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im return null; return new BasicMachineMetadata(node.getId(), node.getName(), ((node instanceof NodeMetadata) ? Iterators.tryFind( ((NodeMetadata)node).getPublicAddresses().iterator(), Predicates.alwaysTrue() ).orNull() : null), - ((node instanceof NodeMetadata) ? ((NodeMetadata)node).getStatus()==Status.RUNNING : null), + ((node instanceof NodeMetadata) ? toMachineStatus( ((NodeMetadata)node).getStatus() ) : null), node); } + public static MachineStatus toMachineStatus(Status status) { + if (status==null) return null; + switch (status) { + case PENDING: return MachineStatus.TRANSITIONING; + case RUNNING: return MachineStatus.RUNNING; + case SUSPENDED: return MachineStatus.SUSPENDED; + case ERROR: return MachineStatus.ERROR; + + case TERMINATED: + case UNRECOGNIZED: + //below + } + return MachineStatus.UNKNOWN; + } + @Override public MachineManagementMixins.MachineMetadata getMachineMetadata(MachineLocation l) { if (l instanceof JcloudsSshMachineLocation) { @@ -1212,7 +1220,7 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im */ @Override public void suspendMachine(MachineLocation rawLocation) { - String instanceId = vmInstanceIds.remove(rawLocation); + String instanceId = vmInstanceIds.get(rawLocation); if (instanceId == null) { LOG.info("Attempt to suspend unknown machine " + rawLocation + " in " + this); throw new IllegalArgumentException("Unknown machine " + rawLocation); @@ -1225,7 +1233,8 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im toThrow = e; LOG.error("Problem suspending machine " + rawLocation + " in " + this + ", instance id " + instanceId, e); } - removeChild(rawLocation); + // before 2020-12 we removed the child; we don't actually want to, as it still exists; and it can trigger a release which could destroy it + //removeChild(rawLocation); if (toThrow != null) { throw Exceptions.propagate(toThrow); } @@ -1236,6 +1245,8 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im * <p/> * Note that this method does <b>not</b> call the lifecycle methods of any * {@link #getCustomizers(ConfigBag) customizers} attached to this location. + * <p/> + * Also note other machines with the same ID may be unmanaged as part of this. * * @param flags See {@link #registerMachine(ConfigBag)} for a description of required fields. * @see #registerMachine(ConfigBag) @@ -1243,17 +1254,33 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im @Override public JcloudsMachineLocation resumeMachine(Map<?, ?> flags) { ConfigBag setup = ConfigBag.newInstanceExtending(config().getBag(), flags); - LOG.info("{} using resuming node matching properties: {}", this, Sanitizer.sanitize(setup)); + LOG.info("Resuming machine in {} matching properties {}", this, Sanitizer.sanitize(setup)); ComputeService computeService = getComputeService(setup); NodeMetadata node = findNodeOrThrow(setup); - LOG.debug("{} resuming {}", this, node); + LOG.debug("{} resuming node {}", this, node); computeService.resumeNode(node.getId()); // Load the node a second time once it is resumed to get an object with // hostname and addresses populated. node = findNodeOrThrow(setup); - LOG.debug("{} resumed {}", this, node); + LOG.debug("{} resumed node {}", this, node); JcloudsMachineLocation registered = registerMachineLocation(setup, node); - LOG.info("{} resumed and registered {}", this, registered); + boolean madeNew = true; + for (Location l : getChildren()) { + if (l instanceof JcloudsMachineLocation && !Boolean.FALSE.equals(MachineLifecycleUtils.isSameInstance((JcloudsMachineLocation)l, registered, false))) { + if (MachineLifecycleUtils.isSameInstance((JcloudsMachineLocation) l, registered, true)) { + // use this machine + if (madeNew) { + removeChild(registered); + madeNew = false; + } + registered = (JcloudsMachineLocation) l; + } else { + // unmanage any old machine location which has no-longer-valid details + removeChild(l); + } + } + } + LOG.info("Resumed {} in {}", registered, this); return registered; } @@ -1264,20 +1291,16 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im } @Override - public MachineLocation startupMachine(MachineLocation l) { - if (!(l instanceof JcloudsSshMachineLocation)) { - throw new IllegalStateException("Cannot startup machine "+l+"; wrong type"); - } - return resumeMachine(ImmutableMap.of("id", ((JcloudsSshMachineLocation)l).getJcloudsId())); + public MachineLocation startupMachine(Map<?, ?> flags) { + return resumeMachine(flags); } @Override - public MachineLocation rebootMachine(MachineLocation l) { + public void rebootMachine(MachineLocation l) { if (!(l instanceof JcloudsSshMachineLocation)) { throw new IllegalStateException("Cannot startup machine "+l+"; wrong type"); } getComputeService().rebootNode(((JcloudsSshMachineLocation)l).getJcloudsId()); - return l; } @Override diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsMachineLocation.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsMachineLocation.java index 83cfc2e..5290a3d 100644 --- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsMachineLocation.java +++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsMachineLocation.java @@ -18,11 +18,16 @@ */ package org.apache.brooklyn.location.jclouds; +import com.google.common.base.Optional; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import org.apache.brooklyn.api.location.MachineLocation; +import org.apache.brooklyn.core.location.MachineLifecycleUtils; import org.apache.brooklyn.location.jclouds.api.JcloudsMachineLocationPublic; +import org.apache.brooklyn.util.collections.MutableList; import org.jclouds.compute.domain.NodeMetadata; -import com.google.common.base.Optional; - public interface JcloudsMachineLocation extends JcloudsMachineLocationPublic { @Override @@ -39,4 +44,5 @@ public interface JcloudsMachineLocation extends JcloudsMachineLocationPublic { */ @Deprecated public NodeMetadata getNode(); + } diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsSshMachineLocation.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsSshMachineLocation.java index 905864f..68dd492 100644 --- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsSshMachineLocation.java +++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsSshMachineLocation.java @@ -18,6 +18,7 @@ */ package org.apache.brooklyn.location.jclouds; +import org.apache.brooklyn.api.location.Location; import static org.apache.brooklyn.location.jclouds.api.JcloudsLocationConfigPublic.USE_MACHINE_PUBLIC_ADDRESS_AS_PRIVATE_ADDRESS; import static org.apache.brooklyn.util.JavaGroovyEquivalents.groovyTruth; @@ -296,7 +297,16 @@ public class JcloudsSshMachineLocation extends SshMachineLocation implements Jcl public JcloudsLocation getParent() { return jcloudsParent; } - + + @Override + public void setParent(Location newParent, boolean updateChildListParents) { + if (newParent==null || newParent instanceof JcloudsLocation) { + // used to clear parent when removing from parent, to prevent releasing it + jcloudsParent = (JcloudsLocation) newParent; + } + super.setParent(newParent, updateChildListParents); + } + @Override public String getHostname() { // Changed behaviour in Brooklyn 0.9.0. Previously it just did node.getHostname(), which diff --git a/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/JcloudsLocationSuspendResumeMachineLiveTest.java b/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/JcloudsLocationSuspendResumeMachineLiveTest.java index dd8865a..f124cfa 100644 --- a/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/JcloudsLocationSuspendResumeMachineLiveTest.java +++ b/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/JcloudsLocationSuspendResumeMachineLiveTest.java @@ -19,17 +19,21 @@ package org.apache.brooklyn.location.jclouds; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - +import com.google.common.collect.ImmutableMap; import org.apache.brooklyn.api.location.MachineLocation; +import org.apache.brooklyn.core.location.MachineLifecycleUtils; +import org.apache.brooklyn.core.location.MachineLifecycleUtils.MachineStatus; +import org.apache.brooklyn.location.ssh.SshMachineLocation; +import org.apache.brooklyn.util.core.config.ConfigBag; +import org.apache.brooklyn.util.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.Assert; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import com.google.common.collect.ImmutableMap; - public class JcloudsLocationSuspendResumeMachineLiveTest extends AbstractJcloudsLiveTest { private static final Logger LOG = LoggerFactory.getLogger(JcloudsLocationSuspendResumeMachineLiveTest.class); @@ -46,17 +50,45 @@ public class JcloudsLocationSuspendResumeMachineLiveTest extends AbstractJclouds @Test(groups = "Live") public void testObtainThenSuspendThenResumeMachine() throws Exception { - MachineLocation machine = obtainMachine(ImmutableMap.of( - "imageId", EUWEST_IMAGE_ID)); + MachineLocation machine = obtainMachine(ConfigBag.newInstance() + .configure(JcloudsLocationConfig.IMAGE_ID, EUWEST_IMAGE_ID) + .configure(JcloudsLocationConfig.OPEN_IPTABLES, false) // optimization + .getAllConfig()); JcloudsSshMachineLocation sshMachine = (JcloudsSshMachineLocation) machine; assertTrue(sshMachine.isSshable(), "Cannot SSH to " + sshMachine); suspendMachine(machine); + ((SshMachineLocation)machine).setSshCheckTimeout(Duration.FIVE_SECONDS); assertFalse(sshMachine.isSshable(), "Should not be able to SSH to suspended machine"); + ((SshMachineLocation)machine).setSshCheckTimeout(null); MachineLocation machine2 = resumeMachine(ImmutableMap.of("id", sshMachine.getJcloudsId())); assertTrue(machine2 instanceof JcloudsSshMachineLocation); assertTrue(((JcloudsSshMachineLocation) machine2).isSshable(), "Cannot SSH to " + machine2); } + @Test(groups = "Live") + public void testObtainThenShutdownThenRestart() throws Exception { + MachineLocation machine = obtainMachine(ConfigBag.newInstance() + .configure(JcloudsLocationConfig.IMAGE_ID, EUWEST_IMAGE_ID) + .configure(JcloudsLocationConfig.OPEN_IPTABLES, false) // optimization + .getAllConfig()); + JcloudsSshMachineLocation sshMachine = (JcloudsSshMachineLocation) machine; + Assert.assertEquals(new MachineLifecycleUtils(sshMachine).getStatus(), MachineStatus.RUNNING); + assertTrue(sshMachine.isSshable(), "Cannot SSH to " + sshMachine); + + jcloudsLocation.shutdownMachine(sshMachine); + sshMachine.setSshCheckTimeout(Duration.FIVE_SECONDS); + assertFalse(sshMachine.isSshable(), "Should not be able to SSH to suspended machine"); + + Assert.assertEquals(new MachineLifecycleUtils(sshMachine).exists(), Boolean.TRUE); + Assert.assertEquals(new MachineLifecycleUtils(sshMachine).getStatus(), MachineStatus.SUSPENDED); // shutdown suspends in AWS + sshMachine.setSshCheckTimeout(null); + + MachineLocation machine2 = new MachineLifecycleUtils(sshMachine).makeRunning().get(); + assertTrue(machine2 instanceof JcloudsSshMachineLocation); + assertTrue(((JcloudsSshMachineLocation) machine2).isSshable(), "Cannot SSH to " + machine2); + Assert.assertEquals(new MachineLifecycleUtils(machine2).getStatus(), MachineStatus.RUNNING); + } + } diff --git a/logging/logback-includes/src/main/resources/brooklyn/logback-logger-excludes.xml b/logging/logback-includes/src/main/resources/brooklyn/logback-logger-excludes.xml index f4b4644..f488196 100644 --- a/logging/logback-includes/src/main/resources/brooklyn/logback-logger-excludes.xml +++ b/logging/logback-includes/src/main/resources/brooklyn/logback-logger-excludes.xml @@ -56,6 +56,10 @@ (Turn them back on if you need to see how API-doc gets generated, and also see https://github.com/wordnik/swagger-core/issues/58) --> </logger> + <!-- Gives spurious warnings --> + <logger name="org.jclouds.location.suppliers.implicit.GetRegionIdMatchingProviderURIOrNull" level="ERROR" additivity="false"> + <appender-ref ref="FILE" /> + </logger> <!-- The MongoDB Java driver is much too noisy at INFO. --> <logger name="org.mongodb.driver" level="WARN" additivity="false"> <appender-ref ref="FILE" /> diff --git a/software/base/src/main/java/org/apache/brooklyn/entity/software/base/lifecycle/MachineLifecycleEffectorTasks.java b/software/base/src/main/java/org/apache/brooklyn/entity/software/base/lifecycle/MachineLifecycleEffectorTasks.java index ec64c7e..8807d00 100644 --- a/software/base/src/main/java/org/apache/brooklyn/entity/software/base/lifecycle/MachineLifecycleEffectorTasks.java +++ b/software/base/src/main/java/org/apache/brooklyn/entity/software/base/lifecycle/MachineLifecycleEffectorTasks.java @@ -364,7 +364,7 @@ public abstract class MachineLifecycleEffectorTasks { // Opportunity to block startup until other dependent components are available try (CloseableLatch latch = waitForCloseableLatch(entity(), SoftwareProcess.START_LATCH)) { - preStartAtMachineAsync(locationSF); + preStartAtMachineAsync(locationSF, parameters); DynamicTasks.queue("start (processes)", new StartProcessesAtMachineTask(locationSF)); postStartAtMachineAsync(parameters); } @@ -452,8 +452,8 @@ public abstract class MachineLifecycleEffectorTasks { /** * Wraps a call to {@link #preStartCustom(MachineLocation, ConfigBag)}, after setting the hostname and address. */ - protected void preStartAtMachineAsync(final Supplier<MachineLocation> machineS) { - DynamicTasks.queue("pre-start", new PreStartTask(machineS.get())); + protected void preStartAtMachineAsync(final Supplier<MachineLocation> machineS, ConfigBag parameters) { + DynamicTasks.queue("pre-start", new PreStartTask(machineS.get(), parameters)); } protected class PreStartTask implements Runnable {