Author: asavu
Date: Mon Jan 2 13:48:51 2012
New Revision: 1226416
URL: http://svn.apache.org/viewvc?rev=1226416&view=rev
Log:
WHIRR-332. Need to specify different instance size/type depending on role
(asavu)
Modified:
whirr/trunk/CHANGES.txt
whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/LaunchClusterCommandTest.java
whirr/trunk/core/src/main/java/org/apache/whirr/ClusterSpec.java
whirr/trunk/core/src/main/java/org/apache/whirr/InstanceTemplate.java
whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java
whirr/trunk/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java
whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java
whirr/trunk/core/src/test/java/org/apache/whirr/ClusterSpecTest.java
whirr/trunk/core/src/test/java/org/apache/whirr/actions/BootstrapClusterActionTest.java
whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java
whirr/trunk/recipes/hadoop-ec2.properties
Modified: whirr/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/whirr/trunk/CHANGES.txt?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
--- whirr/trunk/CHANGES.txt (original)
+++ whirr/trunk/CHANGES.txt Mon Jan 2 13:48:51 2012
@@ -11,6 +11,9 @@ Trunk (unreleased changes)
WHIRR-458. Remove deprecated code and aliasing mechanism (asavu)
+ WHIRR-332. Need to specify different instance size/type
+ depending on role (asavu)
+
BUG FIXES
WHIRR-367. Wrong groupId for zookeeper (Joe Crobak via asavu)
Modified:
whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/LaunchClusterCommandTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/LaunchClusterCommandTest.java?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
---
whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/LaunchClusterCommandTest.java
(original)
+++
whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/LaunchClusterCommandTest.java
Mon Jan 2 13:48:51 2012
@@ -26,9 +26,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -105,8 +103,8 @@ public class LaunchClusterCommandTest {
ClusterSpec expectedClusterSpec = ClusterSpec.withTemporaryKeys(conf);
expectedClusterSpec.setInstanceTemplates(Lists.newArrayList(
- new InstanceTemplate(1, ImmutableSet.of("role1", "role2")),
- new InstanceTemplate(2, ImmutableSet.of("role3"))
+ InstanceTemplate.builder().numberOfInstance(1).roles("role1",
"role2").build(),
+ InstanceTemplate.builder().numberOfInstance(2).roles("role3").build()
));
expectedClusterSpec.setServiceName("test-service");
expectedClusterSpec.setProvider("rackspace");
@@ -139,8 +137,8 @@ public class LaunchClusterCommandTest {
int rc = command.run(null, out, null, Lists.newArrayList(
"--service-name", "hadoop",
"--cluster-name", "test-cluster",
- "--instance-templates", "1 jt+nn,3 dn+tt",
- "--instance-templates-max-percent-failures", "60 dn+tt",
+ "--instance-templates", "1 hadoop-namenode+hadoop-jobtracker,3
hadoop-datanode+hadoop-tasktracker",
+ "--instance-templates-max-percent-failures", "60
hadoop-datanode+hadoop-tasktracker",
"--provider", "ec2",
"--identity", "myusername", "--credential", "mypassword",
"--private-key-file", keys.get("private").getAbsolutePath(),
@@ -151,12 +149,14 @@ public class LaunchClusterCommandTest {
Configuration conf = new PropertiesConfiguration();
conf.addProperty("whirr.version", "version-string");
- conf.addProperty("whirr.instance-templates-max-percent-failure", "60
dn+tt");
+ conf.addProperty("whirr.instance-templates-max-percent-failure", "60
hadoop-datanode+hadoop-tasktracker");
ClusterSpec expectedClusterSpec = ClusterSpec.withTemporaryKeys(conf);
expectedClusterSpec.setInstanceTemplates(Lists.newArrayList(
- new InstanceTemplate(1, 1, Sets.newHashSet("jt", "nn")),
- new InstanceTemplate(3, 2, Sets.newHashSet("dn", "tt"))
+ InstanceTemplate.builder().numberOfInstance(1).minNumberOfInstances(1)
+ .roles("hadoop-namenode", "hadoop-jobtracker").build(),
+ InstanceTemplate.builder().numberOfInstance(3).minNumberOfInstances(2)
+ .roles("hadoop-datanode", "hadoop-tasktracker").build()
));
expectedClusterSpec.setServiceName("hadoop");
expectedClusterSpec.setProvider("ec2");
Modified: whirr/trunk/core/src/main/java/org/apache/whirr/ClusterSpec.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterSpec.java?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
--- whirr/trunk/core/src/main/java/org/apache/whirr/ClusterSpec.java (original)
+++ whirr/trunk/core/src/main/java/org/apache/whirr/ClusterSpec.java Mon Jan 2
13:48:51 2012
@@ -141,7 +141,7 @@ public class ClusterSpec {
HARDWARE_MIN_RAM(Integer.class, false, "The minimum amount of " +
"instance memory. E.g. 1024"),
-
+
LOCATION_ID(String.class, false, "The location to launch " +
"instances in. If not specified then an arbitrary location " +
"will be chosen."),
Modified: whirr/trunk/core/src/main/java/org/apache/whirr/InstanceTemplate.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/InstanceTemplate.java?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
--- whirr/trunk/core/src/main/java/org/apache/whirr/InstanceTemplate.java
(original)
+++ whirr/trunk/core/src/main/java/org/apache/whirr/InstanceTemplate.java Mon
Jan 2 13:48:51 2012
@@ -18,53 +18,101 @@
package org.apache.whirr;
import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Sets.newLinkedHashSet;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import com.google.common.collect.Sets;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* This class describes the type of instances that should be in the cluster.
* This is done by specifying the number of instances in each role.
*/
public class InstanceTemplate {
- private static final Logger LOG =
LoggerFactory.getLogger(InstanceTemplate.class);
-
- private Set<String> roles;
- private int numberOfInstances;
- private int minNumberOfInstances; // some instances may fail, at least a
minimum number is required
- public InstanceTemplate(int numberOfInstances, String... roles) {
- this(numberOfInstances, numberOfInstances,
Sets.newLinkedHashSet(Lists.newArrayList(roles)));
+ public static Builder builder() {
+ return new Builder();
}
- public InstanceTemplate(int numberOfInstances, Set<String> roles) {
- this(numberOfInstances, numberOfInstances, roles);
- }
+ public static class Builder {
+ private int numberOfInstances = -1;
+ private int minNumberOfInstances = -1;
+ private String hardwareId;
+ private String imageId;
+ private Set<String> roles;
+
+ public Builder numberOfInstance(int numberOfInstances) {
+ this.numberOfInstances = numberOfInstances;
+ return this;
+ }
+
+ public Builder minNumberOfInstances(int minNumberOfInstances) {
+ this.minNumberOfInstances = minNumberOfInstances;
+ return this;
+ }
+
+ public Builder hardwareId(String hardwareId) {
+ this.hardwareId = hardwareId;
+ return this;
+ }
- public InstanceTemplate(int numberOfInstances, int minNumberOfInstances,
String... roles) {
- this(numberOfInstances, minNumberOfInstances,
Sets.newLinkedHashSet(Lists.newArrayList(roles)));
+ public Builder imageId(String imageId) {
+ this.imageId = imageId;
+ return this;
+ }
+
+ public Builder roles(String... roles) {
+ this.roles = newLinkedHashSet(newArrayList(roles));
+ return this;
+ }
+
+ public Builder roles(Set<String> roles) {
+ this.roles = newLinkedHashSet(roles);
+ return this;
+ }
+
+ public InstanceTemplate build() {
+ if (minNumberOfInstances == -1) {
+ minNumberOfInstances = numberOfInstances;
+ }
+ return new InstanceTemplate(numberOfInstances, minNumberOfInstances,
roles, hardwareId, imageId);
+ }
}
- public InstanceTemplate(int numberOfInstances, int minNumberOfInstances,
Set<String> roles) {
+ private int numberOfInstances;
+ private int minNumberOfInstances; // some instances may fail, at least a
minimum number is required
+ private String hardwareId;
+ private String imageId;
+ private Set<String> roles;
+
+
+ private InstanceTemplate(int numberOfInstances, int minNumberOfInstances,
+ Set<String> roles, String hardwareId, String
imageId) {
for (String role : roles) {
checkArgument(!StringUtils.contains(role, " "),
- "Role '%s' may not contain space characters.", role);
+ "Role '%s' may not contain space characters.", role);
}
- this.roles = roles;
this.numberOfInstances = numberOfInstances;
this.minNumberOfInstances = minNumberOfInstances;
+ this.hardwareId = hardwareId;
+ this.imageId = imageId;
+ this.roles = roles;
}
public Set<String> getRoles() {
@@ -79,58 +127,80 @@ public class InstanceTemplate {
return minNumberOfInstances;
}
+ public String getHardwareId() {
+ return hardwareId;
+ }
+
+ public String getImageId() {
+ return imageId;
+ }
+
public boolean equals(Object o) {
if (o instanceof InstanceTemplate) {
InstanceTemplate that = (InstanceTemplate) o;
- return Objects.equal(numberOfInstances, that.numberOfInstances)
- && Objects.equal(minNumberOfInstances, that.minNumberOfInstances)
+ return numberOfInstances == that.numberOfInstances
+ && minNumberOfInstances == that.minNumberOfInstances
+ && Objects.equal(hardwareId, that.hardwareId)
+ && Objects.equal(imageId, that.imageId)
&& Objects.equal(roles, that.roles);
}
return false;
}
public int hashCode() {
- return Objects.hashCode(numberOfInstances, minNumberOfInstances, roles);
+ return Objects.hashCode(numberOfInstances, minNumberOfInstances,
+ hardwareId, imageId, roles);
}
public String toString() {
return Objects.toStringHelper(this)
.add("numberOfInstances", numberOfInstances)
.add("minNumberOfInstances", minNumberOfInstances)
+ .add("hardwareId", hardwareId)
+ .add("imageId", imageId)
.add("roles", roles)
.toString();
}
public static Map<String, String> parse(String... strings) {
- Set<String> roles = Sets.newLinkedHashSet(Lists.newArrayList(strings));
+ Set<String> roles = newLinkedHashSet(newArrayList(strings));
Map<String, String> templates = Maps.newHashMap();
for (String s : roles) {
String[] parts = s.split(" ");
checkArgument(parts.length == 2,
- "Invalid instance template syntax for '%s'. Does not match " +
+ "Invalid instance template syntax for '%s'. Does not match " +
"'<number> <role1>+<role2>+<role3>...', e.g. '1
hadoop-namenode+hadoop-jobtracker'.", s);
templates.put(parts[1], parts[0]);
}
return templates;
}
- public static List<InstanceTemplate> parse(Configuration cconf) {
- final String[] strings =
cconf.getStringArray(ClusterSpec.Property.INSTANCE_TEMPLATES.getConfigName());
- Map<String, String> maxPercentFailures =
parse(cconf.getStringArray(ClusterSpec.Property.INSTANCE_TEMPLATES_MAX_PERCENT_FAILURES.getConfigName()));
- Map<String, String> minInstances =
parse(cconf.getStringArray(ClusterSpec.Property.INSTANCE_TEMPLATES_MINIMUM_NUMBER_OF_INSTANCES.getConfigName()));
- List<InstanceTemplate> templates = Lists.newArrayList();
+ public static List<InstanceTemplate> parse(Configuration configuration)
throws ConfigurationException {
+ final String[] strings = configuration.getStringArray(
+ ClusterSpec.Property.INSTANCE_TEMPLATES.getConfigName());
+
+ Map<String, String> maxPercentFailures =
parse(configuration.getStringArray(
+
ClusterSpec.Property.INSTANCE_TEMPLATES_MAX_PERCENT_FAILURES.getConfigName()));
+ Map<String, String> minInstances = parse(configuration.getStringArray(
+
ClusterSpec.Property.INSTANCE_TEMPLATES_MINIMUM_NUMBER_OF_INSTANCES.getConfigName()));
+
+ List<InstanceTemplate> templates = newArrayList();
for (String s : strings) {
String[] parts = s.split(" ");
checkArgument(parts.length == 2,
- "Invalid instance template syntax for '%s'. Does not match " +
+ "Invalid instance template syntax for '%s'. Does not match " +
"'<number> <role1>+<role2>+<role3>...', e.g. '1
hadoop-namenode+hadoop-jobtracker'.", s);
+
int num = Integer.parseInt(parts[0]);
int minNumberOfInstances = 0;
- final String maxPercentFail = maxPercentFailures.get(parts[1]);
+ String maxPercentFail = maxPercentFailures.get(parts[1]);
+ String templateGroup = parts[1];
+
if (maxPercentFail != null) {
// round up integer division (a + b -1) / b
minNumberOfInstances = (Integer.parseInt(maxPercentFail) * num + 99) /
100;
}
+
String minNumberOfInst = minInstances.get(parts[1]);
if (minNumberOfInst != null) {
int minExplicitlySet = Integer.parseInt(minNumberOfInst);
@@ -140,11 +210,52 @@ public class InstanceTemplate {
minNumberOfInstances = minExplicitlySet;
}
}
+
if (minNumberOfInstances == 0 || minNumberOfInstances > num) {
minNumberOfInstances = num;
}
- templates.add(new InstanceTemplate(num, minNumberOfInstances,
parts[1].split("\\+")));
+
+ String hardwareId = configuration.getString(
+ "whirr.templates." + templateGroup + ".hardware-id", null);
+ String imageId = configuration.getString(
+ "whirr.templates." + templateGroup + ".image-id", null);
+
+ templates.add(InstanceTemplate.builder()
+ .numberOfInstance(num)
+ .minNumberOfInstances(minNumberOfInstances)
+ .hardwareId(hardwareId)
+ .imageId(imageId)
+ .roles(templateGroup.split("\\+"))
+ .build()
+ );
}
+ validateThatWeHaveNoOtherOverrides(templates, configuration);
return templates;
}
+
+ private static void validateThatWeHaveNoOtherOverrides(
+ List<InstanceTemplate> templates, Configuration configuration) throws
ConfigurationException {
+
+ Set<String> groups = Sets.newHashSet(Iterables.transform(templates,
+ new Function<InstanceTemplate, String>() {
+ private final Joiner plusJoiner = Joiner.on("+");
+ @Override
+ public String apply(InstanceTemplate instance) {
+ return plusJoiner.join(instance.getRoles());
+ }
+ }));
+
+ Pattern pattern = Pattern.compile("^whirr\\.templates\\.([^.]+)\\..*$");
+ Iterator iterator = configuration.getKeys("whirr.templates");
+
+ while(iterator.hasNext()) {
+ String key = String.class.cast(iterator.next());
+ Matcher matcher = pattern.matcher(key);
+
+ if (matcher.find() && !groups.contains(matcher.group(1))) {
+ throw new ConfigurationException(String.format("'%s' is referencing a
" +
+ "template group not present in 'whirr.instance-templates'", key));
+ }
+ }
+ }
}
Modified:
whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
---
whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java
(original)
+++
whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java
Mon Jan 2 13:48:51 2012
@@ -98,7 +98,7 @@ public class BootstrapClusterAction exte
computeServiceContext.getComputeService();
final Template template = BootstrapTemplate.build(clusterSpec,
computeService,
- statementBuilder, entry.getValue().getTemplateBuilderStrategy());
+ statementBuilder, entry.getValue().getTemplateBuilderStrategy(),
entry.getKey());
if (template.getOptions() != null) {
template.getOptions()
Modified:
whirr/trunk/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
---
whirr/trunk/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java
(original)
+++
whirr/trunk/core/src/main/java/org/apache/whirr/compute/BootstrapTemplate.java
Mon Jan 2 13:48:51 2012
@@ -23,6 +23,7 @@ import com.google.common.collect.Immutab
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.InstanceTemplate;
import org.apache.whirr.service.jclouds.StatementBuilder;
import org.apache.whirr.service.jclouds.TemplateBuilderStrategy;
import org.jclouds.aws.ec2.AWSEC2Client;
@@ -53,7 +54,8 @@ public class BootstrapTemplate {
ClusterSpec clusterSpec,
ComputeService computeService,
StatementBuilder statementBuilder,
- TemplateBuilderStrategy strategy
+ TemplateBuilderStrategy strategy,
+ InstanceTemplate instanceTemplate
) throws MalformedURLException {
LOG.info("Configuring template");
@@ -70,7 +72,7 @@ public class BootstrapTemplate {
TemplateBuilder templateBuilder = computeService.templateBuilder()
.options(runScript(runScript));
- strategy.configureTemplateBuilder(clusterSpec, templateBuilder);
+ strategy.configureTemplateBuilder(clusterSpec, templateBuilder,
instanceTemplate);
return setSpotInstancePriceIfSpecified(
computeService.getContext(), clusterSpec, templateBuilder.build()
Modified:
whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
---
whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java
(original)
+++
whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java
Mon Jan 2 13:48:51 2012
@@ -19,8 +19,10 @@
package org.apache.whirr.service.jclouds;
import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.InstanceTemplate;
import org.jclouds.compute.domain.OsFamily;
import org.jclouds.compute.domain.TemplateBuilder;
+import org.jclouds.javax.annotation.Nullable;
/**
* A class to configure a {@link TemplateBuilder}.
@@ -28,10 +30,12 @@ import org.jclouds.compute.domain.Templa
public class TemplateBuilderStrategy {
public void configureTemplateBuilder(ClusterSpec clusterSpec,
- TemplateBuilder templateBuilder) {
+ TemplateBuilder templateBuilder, InstanceTemplate instanceTemplate) {
- if (clusterSpec.getImageId() != null) {
- templateBuilder.imageId(clusterSpec.getImageId());
+ if (clusterSpec.getImageId() != null || instanceTemplate.getImageId() !=
null) {
+ templateBuilder.imageId(
+ or(instanceTemplate.getImageId(), clusterSpec.getImageId())
+ );
} else {
templateBuilder.osFamily(OsFamily.UBUNTU);
templateBuilder.osVersionMatches("10.04");
@@ -41,8 +45,10 @@ public class TemplateBuilderStrategy {
templateBuilder.imageDescriptionMatches("ubuntu-images/");
}
- if (clusterSpec.getHardwareId() != null) {
- templateBuilder.hardwareId(clusterSpec.getHardwareId());
+ if (clusterSpec.getHardwareId() != null ||
instanceTemplate.getHardwareId() != null) {
+ templateBuilder.hardwareId(
+ or(instanceTemplate.getHardwareId(), clusterSpec.getHardwareId())
+ );
} else if(clusterSpec.getHardwareMinRam() != 0) {
templateBuilder.minRam(clusterSpec.getHardwareMinRam());
@@ -55,4 +61,9 @@ public class TemplateBuilderStrategy {
templateBuilder.locationId(clusterSpec.getLocationId());
}
}
+
+ private String or(@Nullable String a, @Nullable String b) {
+ return (a == null) ? b : a;
+ }
+
}
Modified: whirr/trunk/core/src/test/java/org/apache/whirr/ClusterSpecTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/test/java/org/apache/whirr/ClusterSpecTest.java?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
--- whirr/trunk/core/src/test/java/org/apache/whirr/ClusterSpecTest.java
(original)
+++ whirr/trunk/core/src/test/java/org/apache/whirr/ClusterSpecTest.java Mon
Jan 2 13:48:51 2012
@@ -18,7 +18,9 @@
package org.apache.whirr;
+import static com.google.common.collect.Iterables.get;
import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import com.google.common.collect.Iterables;
@@ -47,10 +49,10 @@ import org.junit.Assert;
import org.junit.Test;
public class ClusterSpecTest {
-
+
@Test
public void testDefaultsAreSet()
- throws ConfigurationException, JSchException, IOException {
+ throws ConfigurationException, JSchException, IOException {
ClusterSpec spec = ClusterSpec.withTemporaryKeys();
assertThat(spec.getClusterUser(),
is(System.getProperty("user.name")));
@@ -59,27 +61,27 @@ public class ClusterSpecTest {
@Test
public void testDefaultsCanBeOverridden()
- throws ConfigurationException, JSchException, IOException {
+ throws ConfigurationException, JSchException, IOException {
Configuration conf = new PropertiesConfiguration();
conf.setProperty(ClusterSpec.Property.RUN_URL_BASE.getConfigName(),
- "http://example.org";);
+ "http://example.org";);
ClusterSpec spec = ClusterSpec.withNoDefaults(conf);
assertThat(spec.getRunUrlBase(), is("http://example.org";));
}
@Test
public void testLoginUserSetsSystemProperty()
- throws ConfigurationException {
+ throws ConfigurationException {
Configuration conf = new PropertiesConfiguration();
conf.setProperty(ClusterSpec.Property.LOGIN_USER.getConfigName(),
- "ubuntu");
+ "ubuntu");
ClusterSpec.withNoDefaults(conf);
assertThat(System.getProperty("whirr.login-user"), is("ubuntu"));
}
-
+
@Test
public void testGetConfigurationForKeysWithPrefix()
- throws ConfigurationException, JSchException, IOException {
+ throws ConfigurationException, JSchException, IOException {
Configuration conf = new PropertiesConfiguration();
conf.setProperty("a.b", 1);
conf.setProperty("b.a", 2);
@@ -95,7 +97,7 @@ public class ClusterSpecTest {
assertThat(prefixKeys.get(0), is("a.b"));
assertThat(prefixKeys.get(1), is("a.c"));
}
-
+
@Test
public void testEnvVariableInterpolation() {
Map<String, String> envMap = System.getenv();
@@ -111,12 +113,12 @@ public class ClusterSpecTest {
assertThat(conf.getString("a"), is(firstEntry.getValue()));
assertThat(conf.getString("b"),
- is(String.format("${env:%s}", undefinedEnvVar)));
+ is(String.format("${env:%s}", undefinedEnvVar)));
}
@Test
public void testDefaultPublicKey()
- throws ConfigurationException, JSchException, IOException {
+ throws ConfigurationException, JSchException, IOException {
Map<String, File> keys = KeyPair.generateTemporaryFiles();
Configuration conf = new PropertiesConfiguration();
@@ -125,17 +127,17 @@ public class ClusterSpecTest {
ClusterSpec spec = ClusterSpec.withNoDefaults(conf);
Assert.assertEquals(IOUtils.toString(
- new FileReader(keys.get("public"))), spec.getPublicKey());
+ new FileReader(keys.get("public"))), spec.getPublicKey());
}
@Test(expected = ConfigurationException.class)
public void testDummyPrivateKey()
- throws JSchException, IOException, ConfigurationException {
+ throws JSchException, IOException, ConfigurationException {
File privateKeyFile = File.createTempFile("private", "key");
privateKeyFile.deleteOnExit();
Files.write(("-----BEGIN RSA PRIVATE KEY-----\n" +
- "DUMMY FILE\n" +
- "-----END RSA PRIVATE KEY-----").getBytes(), privateKeyFile);
+ "DUMMY FILE\n" +
+ "-----END RSA PRIVATE KEY-----").getBytes(), privateKeyFile);
Configuration conf = new PropertiesConfiguration();
conf.setProperty("whirr.private-key-file",
privateKeyFile.getAbsolutePath());
@@ -145,7 +147,7 @@ public class ClusterSpecTest {
@Test(expected = ConfigurationException.class)
public void testEncryptedPrivateKey()
- throws JSchException, IOException, ConfigurationException {
+ throws JSchException, IOException, ConfigurationException {
File privateKey = KeyPair.generateTemporaryFiles("dummy").get("private");
Configuration conf = new PropertiesConfiguration();
@@ -159,7 +161,7 @@ public class ClusterSpecTest {
Configuration conf = new PropertiesConfiguration();
conf.setProperty("whirr.private-key-file",
"/dummy/path/that/does/not/exists");
- ClusterSpec.withNoDefaults(conf);
+ ClusterSpec.withNoDefaults(conf);
}
@Test(expected = ConfigurationException.class)
@@ -197,20 +199,23 @@ public class ClusterSpecTest {
conf.setProperty("whirr.private-key-file",
first.get("private").getAbsolutePath());
conf.setProperty("whirr.public-key-file",
second.get("public").getAbsolutePath());
- ClusterSpec.withNoDefaults(conf);
+ ClusterSpec.withNoDefaults(conf);
}
-
+
@Test(expected = IllegalArgumentException.class)
public void testMissingCommaInInstanceTemplates() throws Exception {
Configuration conf = new PropertiesConfiguration();
conf.setProperty(ClusterSpec.Property.INSTANCE_TEMPLATES.getConfigName(),
- "1 a+b 2 c+d"); // missing comma
+ "1 a+b 2 c+d"); // missing comma
ClusterSpec.withTemporaryKeys(conf);
}
@Test(expected = IllegalArgumentException.class)
public void testRoleMayNotContainSpaces() {
- new InstanceTemplate(1, "a b");
+ InstanceTemplate.builder()
+ .numberOfInstance(1)
+ .minNumberOfInstances(1)
+ .roles("a b").build();
}
@Test(expected = IllegalArgumentException.class)
@@ -225,7 +230,7 @@ public class ClusterSpecTest {
InstanceTemplate t2 = templates.get(1);
assertThat(t2.getMinNumberOfInstances(), is(2));
}
-
+
@Test(expected = NumberFormatException.class)
public void testNumberFormatExceptionOnInstancesTemplates() throws Exception
{
Configuration conf = new PropertiesConfiguration();
@@ -238,7 +243,7 @@ public class ClusterSpecTest {
InstanceTemplate t2 = templates.get(1);
assertThat(t2.getMinNumberOfInstances(), is(2));
}
-
+
@Test
public void testNumberOfInstancesPerTemplate() throws Exception {
Configuration conf = new PropertiesConfiguration();
@@ -250,7 +255,7 @@ public class ClusterSpecTest {
assertThat(t1.getMinNumberOfInstances(), is(1));
InstanceTemplate t2 = templates.get(1);
assertThat(t2.getMinNumberOfInstances(), is(2));
-
+
conf.setProperty("whirr.instance-templates-max-percent-failures", "60
hadoop-datanode+hadoop-tasktracker");
expectedClusterSpec = ClusterSpec.withNoDefaults(conf);
templates = expectedClusterSpec.getInstanceTemplates();
@@ -284,13 +289,13 @@ public class ClusterSpecTest {
@Test
public void testDefaultBlobStoreforComputeProvider() throws Exception {
- for(String pair : new String[]{
- "ec2:aws-s3",
- "aws-ec2:aws-s3",
- "cloudservers:cloudfiles-us",
- "cloudservers-us:cloudfiles-us",
- "cloudservers-uk:cloudfiles-uk"
- }) {
+ for (String pair : new String[]{
+ "ec2:aws-s3",
+ "aws-ec2:aws-s3",
+ "cloudservers:cloudfiles-us",
+ "cloudservers-us:cloudfiles-us",
+ "cloudservers-uk:cloudfiles-uk"
+ }) {
String[] parts = pair.split(":");
Configuration config = new PropertiesConfiguration();
@@ -300,22 +305,22 @@ public class ClusterSpecTest {
assertThat(spec.getBlobStoreProvider(), is(parts[1]));
}
}
-
+
@Test
- public void testApplySubroleAliases() {
+ public void testApplySubroleAliases() throws ConfigurationException {
CompositeConfiguration c = new CompositeConfiguration();
Configuration config = new PropertiesConfiguration();
- config.addProperty("whirr.instance-templates",
- "1 puppet:somepup::pet+something-else, 1 something-else-only");
- c.addConfiguration(config);
+ config.addProperty("whirr.instance-templates",
+ "1 puppet:somepup::pet+something-else, 1 something-else-only");
+ c.addConfiguration(config);
InstanceTemplate template = InstanceTemplate.parse(c).get(0);
Set<String> expected = Sets.newLinkedHashSet(Arrays.asList(new String[]{
- "puppet:somepup::pet", "something-else"}));
+ "puppet:somepup::pet", "something-else"}));
assertThat(template.getRoles(), is(expected));
-
+
InstanceTemplate template2 = InstanceTemplate.parse(c).get(1);
Set<String> expected2 = Sets.newLinkedHashSet(Arrays.asList(new String[]{
- "something-else-only"}));
+ "something-else-only"}));
assertThat(template2.getRoles(), is(expected2));
}
@@ -329,18 +334,52 @@ public class ClusterSpecTest {
assertThat(spec.copy(), is(spec));
assertThat(spec.copy().hashCode(), is(spec.hashCode()));
}
-
+
@Test
public void testFirewallRules() throws Exception {
PropertiesConfiguration conf = new
PropertiesConfiguration("whirr-core-test.properties");
conf.setProperty("whirr.firewall-rules", "8000,8001");
conf.setProperty("whirr.firewall-rules.serviceA", "9000,9001");
- ClusterSpec spec = ClusterSpec.withTemporaryKeys(
- conf);
-
+ ClusterSpec spec = ClusterSpec.withTemporaryKeys(conf);
+
Map<String, List<String>> firewallRules = spec.getFirewallRules();
-
assertThat(firewallRules.get(null).equals(Lists.<String>newArrayList("8000","8001")),
is(true));
-
assertThat(firewallRules.get("serviceA").equals(Lists.<String>newArrayList("9000","9001")),
is(true));
+
assertThat(firewallRules.get(null).equals(Lists.<String>newArrayList("8000",
"8001")), is(true));
+
assertThat(firewallRules.get("serviceA").equals(Lists.<String>newArrayList("9000",
"9001")), is(true));
+ }
+
+ @Test
+ public void testHardwareIdPerInstanceTemplate() throws Exception {
+ PropertiesConfiguration conf = new
PropertiesConfiguration("whirr-core-test.properties");
+ conf.setProperty("whirr.instance-templates", "2 noop, 1 role1+role2, 1
role1");
+ conf.setProperty("whirr.hardware-id", "c1.xlarge");
+
+ conf.setProperty("whirr.templates.noop.hardware-id", "m1.large");
+ conf.setProperty("whirr.templates.role1+role2.hardware-id", "t1.micro");
+ conf.setProperty("whirr.templates.role1+role2.image-id",
"us-east-1/ami-123324");
+
+ ClusterSpec spec = ClusterSpec.withTemporaryKeys(conf);
+ List<InstanceTemplate> templates = spec.getInstanceTemplates();
+
+ InstanceTemplate noops = get(templates, 0);
+ assert noops.getRoles().contains("noop");
+ assertEquals(noops.getHardwareId(), "m1.large");
+ assertEquals(noops.getImageId(), null);
+
+ InstanceTemplate second = get(templates, 1);
+ assertEquals(second.getHardwareId(), "t1.micro");
+ assertEquals(second.getImageId(), "us-east-1/ami-123324");
+
+ InstanceTemplate third = get(templates, 2);
+ assertEquals(third.getHardwareId(), null);
+ assertEquals(third.getImageId(), null);
}
+ @Test(expected = ConfigurationException.class)
+ public void testInstanceTemplateNotFoundForHardwareId() throws Exception {
+ PropertiesConfiguration conf = new
PropertiesConfiguration("whirr-core-test.properties");
+ conf.setProperty("whirr.instance-templates", "1 role1+role2");
+ conf.setProperty("whirr.templates.role1.hardware-id", "m1.large");
+
+ ClusterSpec.withTemporaryKeys(conf);
+ }
}
Modified:
whirr/trunk/core/src/test/java/org/apache/whirr/actions/BootstrapClusterActionTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/test/java/org/apache/whirr/actions/BootstrapClusterActionTest.java?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
---
whirr/trunk/core/src/test/java/org/apache/whirr/actions/BootstrapClusterActionTest.java
(original)
+++
whirr/trunk/core/src/test/java/org/apache/whirr/actions/BootstrapClusterActionTest.java
Mon Jan 2 13:48:51 2012
@@ -87,7 +87,8 @@ public class BootstrapClusterActionTest
Configuration conf = new PropertiesConfiguration();
conf.addProperty("whirr.service-name", "test-service");
conf.addProperty("whirr.cluster-name", "test-cluster");
- conf.addProperty("whirr.instance-templates", "1
hadoop-namenode+hadoop-jobtracker,4 hadoop-datanode+hadoop-tasktracker");
+ conf.addProperty("whirr.instance-templates",
+ "1 hadoop-namenode+hadoop-jobtracker,4
hadoop-datanode+hadoop-tasktracker");
conf.addProperty("whirr.instance-templates-max-percent-failures", "60
hadoop-datanode+hadoop-tasktracker");
conf.addProperty("whirr.provider", "ec2");
config.addConfiguration(conf);
@@ -152,7 +153,8 @@ public class BootstrapClusterActionTest
Configuration conf = new PropertiesConfiguration();
conf.addProperty("whirr.service-name", "test-service");
conf.addProperty("whirr.cluster-name", "test-cluster");
- conf.addProperty("whirr.instance-templates", "1
hadoop-namenode+hadoop-jobtracker,4 hadoop-datanode+hadoop-tasktracker");
+ conf.addProperty("whirr.instance-templates",
+ "1 hadoop-namenode+hadoop-jobtracker,4
hadoop-datanode+hadoop-tasktracker");
conf.addProperty("whirr.instance-templates-max-percent-failures", "60
hadoop-datanode+hadoop-tasktracker");
conf.addProperty("whirr.provider", "ec2");
config.addConfiguration(conf);
Modified:
whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
---
whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java
(original)
+++
whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java
Mon Jan 2 13:48:51 2012
@@ -20,6 +20,7 @@ package org.apache.whirr.service;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import com.jcraft.jsch.JSchException;
@@ -27,34 +28,38 @@ import java.io.IOException;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.InstanceTemplate;
import org.apache.whirr.service.jclouds.TemplateBuilderStrategy;
+import org.jclouds.compute.domain.OsFamily;
import org.jclouds.compute.domain.TemplateBuilder;
import org.junit.Before;
import org.junit.Test;
public class TemplateBuilderStrategyTest {
-
+
private TemplateBuilderStrategy strategy = new TemplateBuilderStrategy();
+ private InstanceTemplate template;
private ClusterSpec spec;
-
+
@Before
public void setUp() throws ConfigurationException, JSchException,
IOException {
spec = ClusterSpec.withTemporaryKeys();
+ template = mock(InstanceTemplate.class);
}
-
+
@Test
public void testImageIdIsPassedThrough() {
spec.setImageId("my-image-id");
TemplateBuilder builder = mock(TemplateBuilder.class);
- strategy.configureTemplateBuilder(spec, builder);
+ strategy.configureTemplateBuilder(spec, builder, template);
verify(builder).imageId("my-image-id");
}
-
+
@Test
public void testHardwareIdIsPassedThrough() {
spec.setHardwareId("my-hardware-id");
TemplateBuilder builder = mock(TemplateBuilder.class);
- strategy.configureTemplateBuilder(spec, builder);
+ strategy.configureTemplateBuilder(spec, builder, template);
verify(builder).hardwareId("my-hardware-id");
}
@@ -62,8 +67,47 @@ public class TemplateBuilderStrategyTest
public void testLocationIdIsPassedThrough() {
spec.setLocationId("my-location-id");
TemplateBuilder builder = mock(TemplateBuilder.class);
- strategy.configureTemplateBuilder(spec, builder);
+ strategy.configureTemplateBuilder(spec, builder, template);
verify(builder).locationId("my-location-id");
}
+ @Test
+ public void testOverrideHardwareId() {
+ spec.setHardwareId("m1.large");
+ spec.setImageId("us-east-1/ami-333");
+
+ when(template.getHardwareId()).thenReturn("t1.micro");
+
+ TemplateBuilder builder = mock(TemplateBuilder.class);
+ strategy.configureTemplateBuilder(spec, builder, template);
+
+ verify(builder).hardwareId("t1.micro");
+ verify(builder).imageId("us-east-1/ami-333");
+ }
+
+ @Test
+ public void testOverrideImageId() {
+ spec.setHardwareId("m1.large");
+ spec.setImageId("us-east-1/ami-333");
+
+ when(template.getImageId()).thenReturn("us-east-1/ami-111");
+
+ TemplateBuilder builder = mock(TemplateBuilder.class);
+ strategy.configureTemplateBuilder(spec, builder, template);
+
+ verify(builder).hardwareId("m1.large");
+ verify(builder).imageId("us-east-1/ami-111");
+ }
+
+ @Test
+ public void testOverrideOnlyHardwareForInstanceTemplate() {
+ when(template.getHardwareId()).thenReturn("t1.micro");
+
+ TemplateBuilder builder = mock(TemplateBuilder.class);
+ strategy.configureTemplateBuilder(spec, builder, template);
+
+ verify(builder).hardwareId("t1.micro");
+ verify(builder).osFamily(OsFamily.UBUNTU);
+ }
+
}
Modified: whirr/trunk/recipes/hadoop-ec2.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/recipes/hadoop-ec2.properties?rev=1226416&r1=1226415&r2=1226416&view=diff
==============================================================================
--- whirr/trunk/recipes/hadoop-ec2.properties (original)
+++ whirr/trunk/recipes/hadoop-ec2.properties Mon Jan 2 13:48:51 2012
@@ -28,6 +28,10 @@ whirr.cluster-name=hadoop
# Change the number of machines in the cluster here
whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker,5
hadoop-datanode+hadoop-tasktracker
+# Customise instance type or AMI per instance template
+# whirr.templates.hadoop-namenode+hadoop.jobtracker.hardware-id=m1.xlarge
+#
whirr.templates.hadoop-datanode+hadoop.tasktracker.image-id=us-east-1/ami-da0cf8b3
+
# Uncomment out these lines to run CDH
#whirr.hadoop.install-function=install_cdh_hadoop
#whirr.hadoop.configure-function=configure_cdh_hadoop
