Author: asavu
Date: Wed Oct 19 12:42:00 2011
New Revision: 1186173
URL: http://svn.apache.org/viewvc?rev=1186173&view=rev
Log:
WHIRR-243. Allow to run component tests in memory (David Alves, Adrian Cole via
asavu)
Added:
whirr/trunk/core/src/main/java/org/apache/whirr/service/DryRunModule.java
whirr/trunk/core/src/test/java/org/apache/whirr/service/DryRunModuleTest.java
whirr/trunk/core/src/test/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
Modified:
whirr/trunk/CHANGES.txt
whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ConfigureClusterAction.java
whirr/trunk/core/src/main/java/org/apache/whirr/service/ComputeCache.java
Modified: whirr/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/whirr/trunk/CHANGES.txt?rev=1186173&r1=1186172&r2=1186173&view=diff
==============================================================================
--- whirr/trunk/CHANGES.txt (original)
+++ whirr/trunk/CHANGES.txt Wed Oct 19 12:42:00 2011
@@ -37,6 +37,9 @@ Trunk (unreleased changes)
WHIRR-214. First iteration on refactoring the core to support the
addition of nodes to running clusters (asavu)
+ WHIRR-243. Allow to run component tests in memory
+ (David Alves, Adrian Cole via asavu)
+
BUG FIXES
WHIRR-377. Fix broken CLI logging config. (asavu via tomwhite)
Modified:
whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java?rev=1186173&r1=1186172&r2=1186173&view=diff
==============================================================================
---
whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java
(original)
+++
whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java
Wed Oct 19 12:42:00 2011
@@ -19,6 +19,7 @@
package org.apache.whirr.actions;
import com.google.common.base.Function;
+import com.google.common.base.Joiner;
import com.google.common.collect.Collections2;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
@@ -98,7 +99,12 @@ public class BootstrapClusterAction exte
final Template template = BootstrapTemplate.build(clusterSpec,
computeService,
statementBuilder, entry.getValue().getTemplateBuilderStrategy());
-
+
+ if (template.getOptions() != null) {
+ template.getOptions()
+ .nameTask("bootstrap-" +
Joiner.on('_').join(entry.getKey().getRoles()));
+ }
+
Future<Set<? extends NodeMetadata>> nodesFuture = executorService.submit(
new StartupProcess(
clusterSpec.getClusterName(),
Modified:
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ConfigureClusterAction.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/actions/ConfigureClusterAction.java?rev=1186173&r1=1186172&r2=1186173&view=diff
==============================================================================
---
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ConfigureClusterAction.java
(original)
+++
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ConfigureClusterAction.java
Wed Oct 19 12:42:00 2011
@@ -58,6 +58,7 @@ import java.util.concurrent.Future;
import static org.apache.whirr.RolePredicates.onlyRolesIn;
import static
org.jclouds.compute.options.RunScriptOptions.Builder.overrideCredentialsWith;
+
/**
* A {@link org.apache.whirr.ClusterAction} for running a configuration script
on instances
* in the cluster after it has been bootstrapped.
@@ -133,6 +134,7 @@ public class ConfigureClusterAction exte
instance.getId(),
statement,
overrideCredentialsWith(credentials).runAsRoot(true)
+ .nameTask("configure-" +
Joiner.on('_').join(instance.getRoles()))
);
} finally {
Modified:
whirr/trunk/core/src/main/java/org/apache/whirr/service/ComputeCache.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/service/ComputeCache.java?rev=1186173&r1=1186172&r2=1186173&view=diff
==============================================================================
--- whirr/trunk/core/src/main/java/org/apache/whirr/service/ComputeCache.java
(original)
+++ whirr/trunk/core/src/main/java/org/apache/whirr/service/ComputeCache.java
Wed Oct 19 12:42:00 2011
@@ -21,6 +21,7 @@ package org.apache.whirr.service;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Objects;
+import com.google.common.base.Throwables;
import com.google.common.collect.ForwardingObject;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
@@ -71,13 +72,27 @@ public enum ComputeCache implements Func
final Map<Key, ComputeServiceContext> cache = new
MapMaker().makeComputingMap(
new Function<Key, ComputeServiceContext>(){
private final ComputeServiceContextFactory factory = new
ComputeServiceContextFactory();
- private final Set<AbstractModule> wiring = ImmutableSet.of(
- new SshjSshClientModule(),
- new SLF4JLoggingModule(),
- new EnterpriseConfigurationModule(),
- new BindLoginCredentialsPatchForEC2());
+ private Set<AbstractModule> wiring;
+
@Override
public ComputeServiceContext apply(Key arg0) {
+ if (wiring == null) {
+ if (arg0.provider.equals("stub")) {
+ try {
+ wiring = ImmutableSet.of(
+ new SLF4JLoggingModule(),
+ (AbstractModule)
Class.forName("org.apache.whirr.service.DryRunModule").newInstance());
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+ } else {
+ wiring = ImmutableSet.of(
+ new SshjSshClientModule(),
+ new SLF4JLoggingModule(),
+ new EnterpriseConfigurationModule(),
+ new BindLoginCredentialsPatchForEC2());
+ }
+ }
LOG.debug("creating new ComputeServiceContext {}", arg0);
ComputeServiceContext context = new
IgnoreCloseComputeServiceContext(factory.createContext(
arg0.provider, arg0.identity, arg0.credential,
Added: whirr/trunk/core/src/main/java/org/apache/whirr/service/DryRunModule.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/service/DryRunModule.java?rev=1186173&view=auto
==============================================================================
--- whirr/trunk/core/src/main/java/org/apache/whirr/service/DryRunModule.java
(added)
+++ whirr/trunk/core/src/main/java/org/apache/whirr/service/DryRunModule.java
Wed Oct 19 12:42:00 2011
@@ -0,0 +1,353 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service;
+
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.contains;
+import static com.google.common.collect.Iterables.find;
+import static com.google.common.collect.Multimaps.synchronizedListMultimap;
+import static com.google.common.io.ByteStreams.newInputStreamSupplier;
+import static com.google.inject.matcher.Matchers.identicalTo;
+import static com.google.inject.matcher.Matchers.returns;
+import static com.google.inject.matcher.Matchers.subclassesOf;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+import org.jclouds.compute.callables.RunScriptOnNode;
+import org.jclouds.compute.domain.ExecResponse;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.crypto.CryptoStreams;
+import org.jclouds.domain.Credentials;
+import org.jclouds.io.Payload;
+import org.jclouds.io.payloads.StringPayload;
+import org.jclouds.net.IPSocket;
+import org.jclouds.ssh.SshClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.Maps;
+import com.google.common.io.InputSupplier;
+import com.google.inject.AbstractModule;
+
+/**
+ * Outputs orchestration jclouds does to INFO logging and saves an ordered list
+ * of all scripts executed on the cluster that can be used to make assertions.
+ * Use in tests by setting whirr.provider to "stub" and make sure you do not
+ * specify a hardware, image, or location id.
+ *
+ */
+// note that most of this logic will be pulled into jclouds 1.0-beta-10 per
+// http://code.google.com/p/jclouds/issues/detail?id=490
+public class DryRunModule extends AbstractModule {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(DryRunModule.class);
+
+ // an example showing how to intercept any internal method for logging
+ // purposes
+ public class LogCallToRunScriptOnNode implements MethodInterceptor {
+
+ public Object invoke(MethodInvocation i) throws Throwable {
+ if (i.getMethod().getName().equals("call")) {
+ RunScriptOnNode runScriptOnNode = RunScriptOnNode.class.cast(i
+ .getThis());
+ String nodeName = runScriptOnNode.getNode().getName();
+ LOG.info(nodeName + " >> running script");
+ Object returnVal = i.proceed();
+ LOG.info(nodeName + " << " + returnVal);
+ return returnVal;
+ } else {
+ return i.proceed();
+ }
+ }
+ }
+
+ public static synchronized DryRun getDryRun() {
+ return DryRun.INSTANCE;
+ }
+
+ public static void resetDryRun() {
+ DryRun.INSTANCE.executedScripts.clear();
+ }
+
+ // enum singleton pattern
+ public static enum DryRun {
+ INSTANCE;
+ // allow duplicate mappings and use deterministic ordering
+ private final ListMultimap<NodeMetadata, RunScriptOnNode>
executedScripts = synchronizedListMultimap(LinkedListMultimap
+ .<NodeMetadata, RunScriptOnNode> create());
+
+ DryRun() {
+ }
+
+ void newExecution(RunScriptOnNode runScript) {
+ executedScripts.put(runScript.getNode(), runScript);
+ }
+
+ public synchronized ListMultimap<NodeMetadata, RunScriptOnNode>
getExecutions() {
+ return ImmutableListMultimap.copyOf(executedScripts);
+ }
+
+ }
+
+ public class SaveDryRunsByInterceptingRunScriptOnNodeCreation implements
+ MethodInterceptor {
+
+ public Object invoke(MethodInvocation i) throws Throwable {
+ if (i.getMethod().getName().equals("create")) {
+ Object returnVal = i.proceed();
+
getDryRun().newExecution(RunScriptOnNode.class.cast(returnVal));
+ return returnVal;
+ } else {
+ return i.proceed();
+ }
+ }
+ }
+
+ @Override
+ protected void configure() {
+ bind(SshClient.Factory.class).to(LogSshClient.Factory.class);
+ bindInterceptor(subclassesOf(RunScriptOnNode.class),
+ returns(identicalTo(ExecResponse.class)),
+ new LogCallToRunScriptOnNode());
+ bindInterceptor(subclassesOf(RunScriptOnNode.Factory.class),
+ returns(identicalTo(RunScriptOnNode.class)),
+ new SaveDryRunsByInterceptingRunScriptOnNodeCreation());
+ }
+
+ private static class Key {
+ private final IPSocket socket;
+ private final Credentials creds;
+ private final NodeMetadata node;
+
+ Key(IPSocket socket, Credentials creds, @Nullable NodeMetadata node) {
+ this.socket = socket;
+ this.creds = creds;
+ this.node = node;
+ }
+
+ // only the user, not password should be used to identify this
+ // connection
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(socket, creds.identity);
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ return Objects.equal(this.toString(), that.toString());
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s#%s@%s:%d", node.getName(), creds.identity,
+ socket.getAddress(), socket.getPort());
+ }
+ }
+
+ @Singleton
+ private static class LogSshClient implements SshClient {
+ private final Key key;
+
+ public LogSshClient(Key key) {
+ this.key = key;
+ }
+
+ private static class NodeHasAddress implements Predicate<NodeMetadata>
{
+ private final String address;
+
+ private NodeHasAddress(String address) {
+ this.address = address;
+ }
+
+ @Override
+ public boolean apply(NodeMetadata arg0) {
+ return contains(
+ concat(arg0.getPrivateAddresses(),
+ arg0.getPublicAddresses()), address);
+ }
+ }
+
+ @Singleton
+ public static class Factory implements SshClient.Factory {
+
+ // this will ensure only one state per ip socket/user
+ private final Map<Key, SshClient> clientMap;
+ // easy access to node metadata
+ private final ConcurrentMap<String, NodeMetadata> nodes;
+
+ @SuppressWarnings("unused")
+ @Inject
+ public Factory(final ConcurrentMap<String, NodeMetadata> nodes) {
+ this.clientMap = new MapMaker()
+ .makeComputingMap(new Function<Key, SshClient>() {
+
+ @Override
+ public SshClient apply(Key key) {
+ return new LogSshClient(key);
+ }
+
+ });
+ this.nodes = nodes;
+ }
+
+ @Override
+ public SshClient create(final IPSocket socket,
+ Credentials loginCreds) {
+ return clientMap.get(new Key(socket, loginCreds,
+ find(nodes.values(),
+ new NodeHasAddress(socket.getAddress()))));
+ }
+
+ @Override
+ public SshClient create(IPSocket socket, String username,
+ String password) {
+ return create(socket, new Credentials(username, password));
+ }
+
+ @Override
+ public SshClient create(IPSocket socket, String username,
+ byte[] privateKey) {
+ return create(socket, new Credentials(username, new String(
+ privateKey)));
+ }
+ }
+
+ private final Map<String, Payload> contents = Maps.newConcurrentMap();
+
+ @Override
+ public void connect() {
+ LOG.info(toString() + " >> connect()");
+ }
+
+ @Override
+ public void disconnect() {
+ LOG.info(toString() + " >> disconnect()");
+ }
+
+ public ThreadLocal<AtomicInteger> delay = new
ThreadLocal<AtomicInteger>();
+ public static int callDelay = 5;
+
+ @Override
+ public ExecResponse exec(String script) {
+ LOG.info(toString() + " >> exec(" + script + ")");
+ // jclouds checks the status code, but only when seeing if a job
+ // completed. to emulate real scripts all calls are delayed by
+ // forcing
+ // jclouds to call status multiple times (5 by default) before
+ // returning exitCode 1.
+ if (delay.get() == null) {
+ delay.set(new AtomicInteger(0));
+ }
+ ExecResponse exec;
+ if (script.endsWith(" status")) {
+ if (delay.get().get() >= callDelay) {
+ exec = new ExecResponse("", "", 1);
+ } else {
+ exec = new ExecResponse("", "", 0);
+ }
+ } else {
+ exec = new ExecResponse("", "", 0);
+ }
+
+ LOG.info(toString() + " << " + exec);
+
+ delay.get().getAndIncrement();
+ return exec;
+ }
+
+ @Override
+ public Payload get(String path) {
+ LOG.info(toString() + " >> get(" + path + ")");
+ Payload returnVal = contents.get(path);
+ LOG.info(toString() + " << md5[" + md5Hex(returnVal) + "]");
+ return returnVal;
+ }
+
+ @Override
+ public String getHostAddress() {
+ return key.socket.getAddress();
+ }
+
+ @Override
+ public String getUsername() {
+ return key.creds.identity;
+ }
+
+ @Override
+ public void put(String path, Payload payload) {
+ LOG.info(toString() + " >> put(" + path + ", md5["
+ + md5Hex(payload) + "])");
+ contents.put(path, payload);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null)
+ return false;
+ return toString().equals(obj.toString());
+ }
+
+ @Override
+ public int hashCode() {
+ return key.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return key.toString();
+ }
+
+ @Override
+ public void put(String path, String text) {
+ put(path, new StringPayload(text));
+ }
+ }
+
+ public static String md5Hex(String in) {
+ return md5Hex(newInputStreamSupplier(in.getBytes()));
+ }
+
+ public static String md5Hex(InputSupplier<? extends InputStream> supplier)
{
+ try {
+ return CryptoStreams.md5Hex(supplier);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
Added:
whirr/trunk/core/src/test/java/org/apache/whirr/service/DryRunModuleTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/test/java/org/apache/whirr/service/DryRunModuleTest.java?rev=1186173&view=auto
==============================================================================
---
whirr/trunk/core/src/test/java/org/apache/whirr/service/DryRunModuleTest.java
(added)
+++
whirr/trunk/core/src/test/java/org/apache/whirr/service/DryRunModuleTest.java
Wed Oct 19 12:42:00 2011
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service;
+
+import com.google.common.collect.ListMultimap;
+import com.jcraft.jsch.JSchException;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.DryRunModule.DryRun;
+import org.jclouds.compute.callables.RunScriptOnNode;
+import org.jclouds.compute.callables.RunScriptOnNodeAsInitScriptUsingSsh;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.scriptbuilder.InitBuilder;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static junit.framework.Assert.assertFalse;
+
+public class DryRunModuleTest {
+
+ public static class Noop2ClusterActionHandler extends
+ ClusterActionHandlerSupport {
+
+ @Override
+ public String getRole() {
+ return "noop2";
+ }
+
+ }
+
+ public static class Noop3ClusterActionHandler extends
+ ClusterActionHandlerSupport {
+
+ @Override
+ public String getRole() {
+ return "noop3";
+ }
+ }
+
+ /**
+ * Simple test that asserts that a 1 node cluster was launched and
bootstrap
+ * and configure scripts were executed.
+ *
+ * @throws ConfigurationException
+ * @throws IOException
+ * @throws JSchException
+ * @throws InterruptedException
+ */
+ @Test
+ public void testNoInitScriptsAfterConfigurationStarted()
+ throws ConfigurationException, JSchException, IOException,
+ InterruptedException {
+
+ CompositeConfiguration config = new CompositeConfiguration();
+ config.setProperty("whirr.provider", "stub");
+ config.setProperty("whirr.cluster-name", "stub-test");
+ config.setProperty("whirr.instance-templates",
+ "10 noop+noop3,10 noop2+noop,10 noop3+noop2");
+
+ ClusterSpec clusterSpec = ClusterSpec.withTemporaryKeys(config);
+ ClusterController controller = new ClusterController();
+
+ controller.launchCluster(clusterSpec);
+
+ DryRun dryRun = DryRunModule.getDryRun();
+ ListMultimap<NodeMetadata, RunScriptOnNode> list = dryRun
+ .getExecutions();
+
+ // this tests the barrier by making sure that once a configure
+ // script is executed no more setup scripts are executed.
+
+ boolean configStarted = false;
+ for (RunScriptOnNode script : list.values()) {
+ if (!configStarted &&
getScriptName(script).startsWith("configure")) {
+ configStarted = true;
+ continue;
+ }
+ if (configStarted) {
+ assertFalse(
+ "A setup script was executed after the first configure
script.",
+ getScriptName(script).startsWith("setup"));
+ }
+ }
+ }
+
+ private String getScriptName(RunScriptOnNode script) {
+ return ((InitBuilder) ((RunScriptOnNodeAsInitScriptUsingSsh) script)
+ .getStatement()).getInstanceName();
+ }
+
+}
Added:
whirr/trunk/core/src/test/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/test/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler?rev=1186173&view=auto
==============================================================================
---
whirr/trunk/core/src/test/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
(added)
+++
whirr/trunk/core/src/test/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
Wed Oct 19 12:42:00 2011
@@ -0,0 +1,14 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.whirr.service.NoopClusterActionHandler
+org.apache.whirr.service.DryRunModuleTest$Noop2ClusterActionHandler
+org.apache.whirr.service.DryRunModuleTest$Noop3ClusterActionHandler
\ No newline at end of file
