Author: asavu
Date: Mon Nov 7 10:50:52 2011
New Revision: 1198698
URL: http://svn.apache.org/viewvc?rev=1198698&view=rev
Log:
WHIRR-399. Move common script setup and script execution fork/join outside of
ConfigureClusterAction and DestroyClusterAction (David Alves via asavu)
Modified:
whirr/trunk/CHANGES.txt
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ConfigureClusterAction.java
whirr/trunk/core/src/main/java/org/apache/whirr/actions/DestroyClusterAction.java
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java
Modified: whirr/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/whirr/trunk/CHANGES.txt?rev=1198698&r1=1198697&r2=1198698&view=diff
==============================================================================
--- whirr/trunk/CHANGES.txt (original)
+++ whirr/trunk/CHANGES.txt Mon Nov 7 10:50:52 2011
@@ -59,6 +59,9 @@ Trunk (unreleased changes)
WHIRR-398. Implement the execution of scripts on DestroyClusterAction
(David Alves via asavu)
+ WHIRR-399. Move common script setup and script execution fork/join outside
of
+ ConfigureClusterAction and DestroyClusterAction (David Alves via asavu)
+
BUG FIXES
WHIRR-377. Fix broken CLI logging config. (asavu via tomwhite)
Modified:
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ConfigureClusterAction.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/actions/ConfigureClusterAction.java?rev=1198698&r1=1198697&r2=1198698&view=diff
==============================================================================
---
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ConfigureClusterAction.java
(original)
+++
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ConfigureClusterAction.java
Mon Nov 7 10:50:52 2011
@@ -18,45 +18,21 @@
package org.apache.whirr.actions;
-import static org.apache.whirr.RolePredicates.onlyRolesIn;
-import static
org.jclouds.compute.options.RunScriptOptions.Builder.overrideCredentialsWith;
-
import java.io.IOException;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.Map.Entry;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import org.apache.whirr.Cluster;
import org.apache.whirr.ClusterSpec;
import org.apache.whirr.InstanceTemplate;
import org.apache.whirr.RolePredicates;
-import org.apache.whirr.Cluster.Instance;
import org.apache.whirr.service.ClusterActionEvent;
import org.apache.whirr.service.ClusterActionHandler;
import org.apache.whirr.service.FirewallManager.Rule;
-import org.apache.whirr.service.jclouds.StatementBuilder;
-import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceContext;
-import org.jclouds.compute.domain.ExecResponse;
-import org.jclouds.domain.Credentials;
-import org.jclouds.javax.annotation.Nullable;
-import org.jclouds.scriptbuilder.domain.OsFamily;
-import org.jclouds.scriptbuilder.domain.Statement;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
-import com.google.common.base.Joiner;
import com.google.common.collect.Collections2;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
@@ -65,9 +41,6 @@ import com.google.common.primitives.Ints
* in the cluster after it has been bootstrapped.
*/
public class ConfigureClusterAction extends ScriptBasedClusterAction {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ConfigureClusterAction.class);
public ConfigureClusterAction(Function<ClusterSpec, ComputeServiceContext>
getCompute,
Map<String, ClusterActionHandler> handlerMap) {
@@ -78,93 +51,13 @@ public class ConfigureClusterAction exte
protected String getAction() {
return ClusterActionHandler.CONFIGURE_ACTION;
}
-
- @Override
- protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap)
- throws IOException, InterruptedException {
-
- final ExecutorService executorService = Executors.newCachedThreadPool();
- final Collection<Future<ExecResponse>> futures = Sets.newHashSet();
-
- for (Entry<InstanceTemplate, ClusterActionEvent> entry :
eventMap.entrySet()) {
- applyFirewallRules(entry.getValue());
-
- ClusterSpec clusterSpec = entry.getValue().getClusterSpec();
- Cluster cluster = entry.getValue().getCluster();
-
- StatementBuilder statementBuilder =
entry.getValue().getStatementBuilder();
-
- ComputeServiceContext computeServiceContext =
getCompute().apply(clusterSpec);
- final ComputeService computeService =
computeServiceContext.getComputeService();
-
- final Credentials credentials = new Credentials(
- clusterSpec.getClusterUser(),
- clusterSpec.getPrivateKey()
- );
-
- Set<Instance> instances = cluster.getInstancesMatching(
- onlyRolesIn(entry.getKey().getRoles()));
-
- String instanceIds = Joiner.on(", ").join(Iterables.transform(instances,
- new Function<Instance, String>() {
- @Override
- public String apply(@Nullable Instance instance) {
- return instance == null ? "<null>" : instance.getId();
- }
- })
- );
-
- LOG.info("Starting to run configuration scripts on cluster " +
- "instances: {}", instanceIds);
-
- for (final Instance instance : instances) {
- final Statement statement = statementBuilder.build(clusterSpec,
instance);
-
- futures.add(executorService.submit(new Callable<ExecResponse>() {
- @Override
- public ExecResponse call() {
-
- LOG.info("Running configuration script on: {}", instance.getId());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Configuration script for {}:\n{}", instance.getId(),
- statement.render(OsFamily.UNIX));
- }
-
- try {
- return computeService.runScriptOnNode(
- instance.getId(),
- statement,
- overrideCredentialsWith(credentials).runAsRoot(true)
- .nameTask("configure-" +
Joiner.on('_').join(instance.getRoles()))
- );
-
- } finally {
- LOG.info("Configuration script run completed on: {}",
instance.getId());
- }
- }
- }));
- }
- }
-
- for (Future<ExecResponse> future : futures) {
- try {
- ExecResponse execResponse = future.get();
- if (execResponse.getExitCode() != 0) {
- LOG.error("Error running script: {}\n{}", execResponse.getError(),
- execResponse.getOutput());
- }
- } catch (ExecutionException e) {
- throw new IOException(e.getCause());
- }
- }
-
- LOG.info("Finished running configuration scripts on all cluster
instances");
- }
/**
* Apply the firewall rules specified via configuration.
*/
- private void applyFirewallRules(ClusterActionEvent event) throws IOException
{
+ protected void eventSpecificActions(Entry<InstanceTemplate,
ClusterActionEvent> entry)
+ throws IOException {
+ ClusterActionEvent event = entry.getValue();
ClusterSpec clusterSpec = event.getClusterSpec();
Map<String, List<String>> firewallRules = clusterSpec.getFirewallRules();
Modified:
whirr/trunk/core/src/main/java/org/apache/whirr/actions/DestroyClusterAction.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/actions/DestroyClusterAction.java?rev=1198698&r1=1198697&r2=1198698&view=diff
==============================================================================
---
whirr/trunk/core/src/main/java/org/apache/whirr/actions/DestroyClusterAction.java
(original)
+++
whirr/trunk/core/src/main/java/org/apache/whirr/actions/DestroyClusterAction.java
Mon Nov 7 10:50:52 2011
@@ -18,43 +18,21 @@
package org.apache.whirr.actions;
-import static org.apache.whirr.RolePredicates.onlyRolesIn;
-import static
org.jclouds.compute.options.RunScriptOptions.Builder.overrideCredentialsWith;
import static org.jclouds.compute.predicates.NodePredicates.inGroup;
import java.io.IOException;
-import java.util.Collection;
import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import javax.annotation.Nullable;
-
-import org.apache.whirr.Cluster;
-import org.apache.whirr.Cluster.Instance;
import org.apache.whirr.ClusterSpec;
import org.apache.whirr.InstanceTemplate;
import org.apache.whirr.service.ClusterActionEvent;
import org.apache.whirr.service.ClusterActionHandler;
-import org.apache.whirr.service.jclouds.StatementBuilder;
import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceContext;
-import org.jclouds.compute.domain.ExecResponse;
-import org.jclouds.domain.Credentials;
-import org.jclouds.scriptbuilder.domain.OsFamily;
-import org.jclouds.scriptbuilder.domain.Statement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
/**
* A {@link ClusterAction} for tearing down a running cluster and freeing up
all
@@ -77,90 +55,10 @@ public class DestroyClusterAction extend
}
@Override
- protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap)
- throws IOException, InterruptedException {
-
- final ExecutorService executorService = Executors.newCachedThreadPool();
- final Collection<Future<ExecResponse>> futures = Sets.newHashSet();
-
+ protected void postRunScriptsActions(
+ Map<InstanceTemplate, ClusterActionEvent> eventMap) throws IOException {
ClusterSpec clusterSpec = eventMap.values().iterator().next()
.getClusterSpec();
-
- for (Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap
- .entrySet()) {
-
- Cluster cluster = entry.getValue().getCluster();
-
- StatementBuilder statementBuilder = entry.getValue()
- .getStatementBuilder();
-
- ComputeServiceContext computeServiceContext = getCompute().apply(
- clusterSpec);
- final ComputeService computeService = computeServiceContext
- .getComputeService();
-
- final Credentials credentials = new Credentials(
- clusterSpec.getClusterUser(), clusterSpec.getPrivateKey());
-
- Set<Instance> instances = cluster.getInstancesMatching(onlyRolesIn(entry
- .getKey().getRoles()));
-
- String instanceIds = Joiner.on(", ").join(
- Iterables.transform(instances, new Function<Instance, String>() {
- @Override
- public String apply(@Nullable Instance instance) {
- return instance == null ? "<null>" : instance.getId();
- }
- }));
-
- LOG.info("Starting to run destroy scripts on cluster " + "instances: {}",
- instanceIds);
-
- for (final Instance instance : instances) {
- final Statement statement = statementBuilder.build(clusterSpec,
- instance);
-
- futures.add(executorService.submit(new Callable<ExecResponse>() {
- @Override
- public ExecResponse call() {
-
- LOG.info("Running destroy script on: {}", instance.getId());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Destroy script for {}:\n{}", instance.getId(),
- statement.render(OsFamily.UNIX));
- }
-
- try {
- return computeService.runScriptOnNode(
- instance.getId(),
- statement,
- overrideCredentialsWith(credentials)
- .runAsRoot(true)
- .nameTask(
- "destroy-" +
Joiner.on('_').join(instance.getRoles())));
-
- } finally {
- LOG.info("Destroy script run completed on: {}",
instance.getId());
- }
- }
- }));
- }
- }
-
- for (Future<ExecResponse> future : futures) {
- try {
- ExecResponse execResponse = future.get();
- if (execResponse.getExitCode() != 0) {
- LOG.error("Error running script: {}\n{}", execResponse.getError(),
- execResponse.getOutput());
- }
- } catch (ExecutionException e) {
- throw new IOException(e.getCause());
- }
- }
-
- LOG.info("Finished running destroy scripts on all cluster instances.");
-
LOG.info("Destroying " + clusterSpec.getClusterName() + " cluster");
ComputeService computeService = getCompute().apply(clusterSpec)
.getComputeService();
Modified:
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java?rev=1198698&r1=1198697&r2=1198698&view=diff
==============================================================================
---
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java
(original)
+++
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java
Mon Nov 7 10:50:52 2011
@@ -18,15 +18,25 @@
package org.apache.whirr.actions;
-import com.google.common.base.Function;
-import com.google.common.collect.ComputationException;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.whirr.RolePredicates.onlyRolesIn;
+import static
org.jclouds.compute.options.RunScriptOptions.Builder.overrideCredentialsWith;
import java.io.IOException;
+import java.util.Collection;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import javax.annotation.Nullable;
import org.apache.whirr.Cluster;
+import org.apache.whirr.Cluster.Instance;
import org.apache.whirr.ClusterAction;
import org.apache.whirr.ClusterSpec;
import org.apache.whirr.InstanceTemplate;
@@ -34,9 +44,21 @@ import org.apache.whirr.service.ClusterA
import org.apache.whirr.service.ClusterActionHandler;
import org.apache.whirr.service.FirewallManager;
import org.apache.whirr.service.jclouds.StatementBuilder;
+import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceContext;
+import org.jclouds.compute.domain.ExecResponse;
+import org.jclouds.domain.Credentials;
+import org.jclouds.scriptbuilder.domain.OsFamily;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ComputationException;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
/**
* A {@link ClusterAction} that provides the base functionality for running
@@ -44,30 +66,34 @@ import static com.google.common.base.Pre
*/
public abstract class ScriptBasedClusterAction extends ClusterAction {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(ScriptBasedClusterAction.class);
+
private final Map<String, ClusterActionHandler> handlerMap;
-
- protected ScriptBasedClusterAction(Function<ClusterSpec,
ComputeServiceContext> getCompute,
+
+ protected ScriptBasedClusterAction(
+ Function<ClusterSpec, ComputeServiceContext> getCompute,
final Map<String, ClusterActionHandler> handlerMap) {
super(getCompute);
this.handlerMap = checkNotNull(handlerMap, "handlerMap");
}
-
- protected abstract void doAction(Map<InstanceTemplate, ClusterActionEvent>
eventMap)
- throws IOException, InterruptedException;
- public Cluster execute(ClusterSpec clusterSpec, Cluster cluster) throws
IOException, InterruptedException {
-
+ public Cluster execute(ClusterSpec clusterSpec, Cluster cluster)
+ throws IOException, InterruptedException {
+
Map<InstanceTemplate, ClusterActionEvent> eventMap = Maps.newHashMap();
Cluster newCluster = cluster;
for (InstanceTemplate instanceTemplate :
clusterSpec.getInstanceTemplates()) {
StatementBuilder statementBuilder = new StatementBuilder();
- ComputeServiceContext computeServiceContext =
getCompute().apply(clusterSpec);
- FirewallManager firewallManager = new
FirewallManager(computeServiceContext,
- clusterSpec, newCluster);
+ ComputeServiceContext computeServiceContext = getCompute().apply(
+ clusterSpec);
+ FirewallManager firewallManager = new FirewallManager(
+ computeServiceContext, clusterSpec, newCluster);
ClusterActionEvent event = new ClusterActionEvent(getAction(),
- clusterSpec, instanceTemplate, newCluster, statementBuilder,
getCompute(), firewallManager);
+ clusterSpec, instanceTemplate, newCluster, statementBuilder,
+ getCompute(), firewallManager);
eventMap.put(instanceTemplate, event);
for (String role : instanceTemplate.getRoles()) {
@@ -77,9 +103,9 @@ public abstract class ScriptBasedCluster
// cluster may have been updated by handler
newCluster = event.getCluster();
}
-
+
doAction(eventMap);
-
+
// cluster may have been updated by action
newCluster = Iterables.get(eventMap.values(), 0).getCluster();
@@ -93,10 +119,115 @@ public abstract class ScriptBasedCluster
newCluster = event.getCluster();
}
}
-
+
return newCluster;
}
+ protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap)
+ throws InterruptedException, IOException {
+ runScripts(eventMap);
+ postRunScriptsActions(eventMap);
+ }
+
+ protected void runScripts(Map<InstanceTemplate, ClusterActionEvent> eventMap)
+ throws InterruptedException, IOException {
+
+ final String phaseName = getAction();
+ final ExecutorService executorService = Executors.newCachedThreadPool();
+ final Collection<Future<ExecResponse>> futures = Sets.newHashSet();
+
+ ClusterSpec clusterSpec = eventMap.values().iterator().next()
+ .getClusterSpec();
+
+ ComputeServiceContext computeServiceContext = getCompute().apply(
+ clusterSpec);
+ final ComputeService computeService = computeServiceContext
+ .getComputeService();
+
+ final Credentials credentials = new Credentials(
+ clusterSpec.getClusterUser(), clusterSpec.getPrivateKey());
+
+ for (Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap
+ .entrySet()) {
+
+ eventSpecificActions(entry);
+
+ Cluster cluster = entry.getValue().getCluster();
+
+ StatementBuilder statementBuilder = entry.getValue()
+ .getStatementBuilder();
+
+ Set<Instance> instances = cluster.getInstancesMatching(onlyRolesIn(entry
+ .getKey().getRoles()));
+
+ String instanceIds = Joiner.on(", ").join(
+ Iterables.transform(instances, new Function<Instance, String>() {
+ @Override
+ public String apply(@Nullable Instance instance) {
+ return instance == null ? "<null>" : instance.getId();
+ }
+ }));
+
+ LOG.info("Starting to run scripts on cluster for phase {}"
+ + "instances: {}", phaseName, instanceIds);
+
+ for (final Instance instance : instances) {
+ final Statement statement = statementBuilder.build(clusterSpec,
+ instance);
+
+ futures.add(executorService.submit(new Callable<ExecResponse>() {
+ @Override
+ public ExecResponse call() {
+
+ LOG.info("Running {} phase script on: {}", phaseName,
+ instance.getId());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{} phase script on: {}\n{}", new Object[] { phaseName,
+ instance.getId(), statement.render(OsFamily.UNIX) });
+ }
+
+ try {
+ return computeService.runScriptOnNode(
+ instance.getId(),
+ statement,
+ overrideCredentialsWith(credentials).runAsRoot(true)
+ .nameTask(
+ phaseName + "-"
+ + Joiner.on('_').join(instance.getRoles())));
+ } finally {
+ LOG.info("{} phase script run completed on: {}", phaseName,
+ instance.getId());
+ }
+ }
+ }));
+ }
+ }
+
+ for (Future<ExecResponse> future : futures) {
+ try {
+ ExecResponse execResponse = future.get();
+ if (execResponse.getExitCode() != 0) {
+ LOG.error("Error running script: {}\n{}", execResponse.getError(),
+ execResponse.getOutput());
+ }
+ } catch (ExecutionException e) {
+ throw new IOException(e.getCause());
+ }
+ }
+
+ executorService.shutdown();
+ LOG.info("Finished running {} phase scripts on all cluster instances",
+ phaseName);
+ }
+
+ protected void eventSpecificActions(
+ Entry<InstanceTemplate, ClusterActionEvent> entry) throws IOException {
+ };
+
+ protected void postRunScriptsActions(
+ Map<InstanceTemplate, ClusterActionEvent> eventMap) throws IOException {
+ };
+
/**
* Try to get an {@see ClusterActionHandler } instance or throw an
* IllegalArgumentException if not found for this role name
