Author: asavu
Date: Fri Mar 9 16:43:33 2012
New Revision: 1298911
URL: http://svn.apache.org/viewvc?rev=1298911&view=rev
Log:
WHIRR-407. Upgrade Cassandra to 1.0.6 (Nathan Milford via Andrei Savu)
Modified:
whirr/branches/branch-0.7/CHANGES.txt
whirr/branches/branch-0.7/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
whirr/branches/branch-0.7/recipes/cassandra-ec2.properties
whirr/branches/branch-0.7/services/cassandra/pom.xml
whirr/branches/branch-0.7/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java
whirr/branches/branch-0.7/services/cassandra/src/main/resources/functions/configure_cassandra.sh
whirr/branches/branch-0.7/services/cassandra/src/main/resources/functions/install_cassandra.sh
whirr/branches/branch-0.7/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandlerTest.java
whirr/branches/branch-0.7/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/integration/CassandraServiceTest.java
whirr/branches/branch-0.7/services/cassandra/src/test/resources/whirr-cassandra-test.properties
Modified: whirr/branches/branch-0.7/CHANGES.txt
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/CHANGES.txt?rev=1298911&r1=1298910&r2=1298911&view=diff
==============================================================================
--- whirr/branches/branch-0.7/CHANGES.txt (original)
+++ whirr/branches/branch-0.7/CHANGES.txt Fri Mar 9 16:43:33 2012
@@ -12,6 +12,8 @@ Release 0.7.2
WHIRR-550. Upgrade Mahout service to 0.6 (Frank Scholten via asavu)
+ WHIRR-407. Upgrade Cassandra to 1.0.6 (Nathan Milford via Andrei Savu)
+
BUG FIXES
Release 0.7.1 - 2012-02-23
Modified:
whirr/branches/branch-0.7/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java?rev=1298911&r1=1298910&r2=1298911&view=diff
==============================================================================
---
whirr/branches/branch-0.7/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
(original)
+++
whirr/branches/branch-0.7/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
Fri Mar 9 16:43:33 2012
@@ -40,7 +40,7 @@ import java.util.Map;
import static org.jclouds.scriptbuilder.domain.Statements.exec;
public class StatementBuilder {
-
+
private static final Logger LOG =
LoggerFactory.getLogger(StatementBuilder.class);
@@ -48,25 +48,48 @@ public class StatementBuilder {
private ClusterSpec clusterSpec;
private Instance instance;
-
+
public ConsolidatedStatement(ClusterSpec clusterSpec, Instance instance) {
this.clusterSpec = clusterSpec;
this.instance = instance;
}
-
+
@Override
public Iterable<String> functionDependencies(OsFamily family) {
- List<String> functions = Lists.newArrayList();
- for (Statement statement : statements) {
- Iterables.addAll(functions, statement.functionDependencies(family));
- }
- return functions;
+ List<String> functions = Lists.newArrayList();
+ for (Statement statement : statements) {
+ Iterables.addAll(functions, statement.functionDependencies(family));
+ }
+ return functions;
}
@Override
public String render(OsFamily family) {
ScriptBuilder scriptBuilder = new ScriptBuilder();
Map<String, String> metadataMap = Maps.newLinkedHashMap();
+
+ addEnvironmentVariablesFromClusterSpec(metadataMap);
+ addDefaultEnvironmentVariablesForInstance(metadataMap, instance);
+ metadataMap.putAll(exports);
+ addPerInstanceCustomEnvironmentVariables(metadataMap, instance);
+
+ String writeVariableExporters =
Utils.writeVariableExporters(metadataMap, family);
+ scriptBuilder.addStatement(exec(writeVariableExporters));
+
+ for (Statement statement : statements) {
+ scriptBuilder.addStatement(statement);
+ }
+
+ return scriptBuilder.render(family);
+ }
+
+ private void addPerInstanceCustomEnvironmentVariables(Map<String, String>
metadataMap, Instance instance) {
+ if (instance != null &&
exportsByInstanceId.containsKey(instance.getId())) {
+ metadataMap.putAll(exportsByInstanceId.get(instance.getId()));
+ }
+ }
+
+ private void addDefaultEnvironmentVariablesForInstance(Map<String, String>
metadataMap, Instance instance) {
metadataMap.putAll(
ImmutableMap.of(
"clusterName", clusterSpec.getClusterName(),
@@ -94,43 +117,46 @@ public class StatementBuilder {
}
}
}
+ }
+
+ private void addEnvironmentVariablesFromClusterSpec(Map<String, String>
metadataMap) {
for (Iterator<?> it =
clusterSpec.getConfiguration().getKeys("whirr.env"); it.hasNext(); ) {
- String key = (String)it.next();
+ String key = (String) it.next();
String value = clusterSpec.getConfiguration().getString(key);
metadataMap.put(key.substring("whirr.env.".length()), value);
}
- metadataMap.putAll(exports);
-
- // Write export statements out directly
- // Using InitBuilder would be a possible improvement
- String writeVariableExporters =
Utils.writeVariableExporters(metadataMap, family);
- scriptBuilder.addStatement(exec(writeVariableExporters));
- for (Statement statement : statements) {
- scriptBuilder.addStatement(statement);
- }
-
- return scriptBuilder.render(family);
}
}
-
+
protected List<Statement> statements = Lists.newArrayList();
- protected Map<String,String> exports = Maps.newLinkedHashMap();
-
+ protected Map<String, String> exports = Maps.newLinkedHashMap();
+ protected Map<String, Map<String, String>> exportsByInstanceId =
Maps.newHashMap();
+
public void addStatement(Statement statement) {
if (!statements.contains(statement)) {
statements.add(statement);
}
}
-
+
public void addStatements(Statement... statements) {
for (Statement statement : statements) {
addStatement(statement);
}
}
-
+
public void addExport(String key, String value) {
exports.put(key, value);
}
+
+ public void addExportPerInstance(String instanceId, String key, String
value) {
+ if (exportsByInstanceId.containsKey(instanceId)) {
+ exportsByInstanceId.get(instanceId).put(key, value);
+ } else {
+ Map<String, String> pairs = Maps.newHashMap();
+ pairs.put(key, value);
+ exportsByInstanceId.put(instanceId, pairs);
+ }
+ }
public Statement build(ClusterSpec clusterSpec) {
return build(clusterSpec, null);
Modified: whirr/branches/branch-0.7/recipes/cassandra-ec2.properties
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/recipes/cassandra-ec2.properties?rev=1298911&r1=1298910&r2=1298911&view=diff
==============================================================================
--- whirr/branches/branch-0.7/recipes/cassandra-ec2.properties (original)
+++ whirr/branches/branch-0.7/recipes/cassandra-ec2.properties Fri Mar 9
16:43:33 2012
@@ -38,5 +38,5 @@ whirr.credential=${env:AWS_SECRET_ACCESS
# whirr.public-key-file=${whirr.private-key-file}.pub
# Expert: specify the version of Cassandra to install.
-#whirr.cassandra.version.major=0.7
-#whirr.cassandra.tarball.url=http://www.apache.org/dist/cassandra/0.7.0/apache-cassandra-0.7.0-bin.tar.gz
+#whirr.cassandra.version.major=1.0.6
+#whirr.cassandra.tarball.url=http://www.apache.org/dist/cassandra/1.0.6/apache-cassandra-1.0.6-bin.tar.gz
Modified: whirr/branches/branch-0.7/services/cassandra/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cassandra/pom.xml?rev=1298911&r1=1298910&r2=1298911&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/cassandra/pom.xml (original)
+++ whirr/branches/branch-0.7/services/cassandra/pom.xml Fri Mar 9 16:43:33
2012
@@ -65,7 +65,7 @@
<dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-all</artifactId>
- <version>0.7.1</version>
+ <version>1.0.6</version>
</dependency>
<dependency>
<groupId>org.apache.cassandra.deps</groupId>
Modified:
whirr/branches/branch-0.7/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java?rev=1298911&r1=1298910&r2=1298911&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java
(original)
+++
whirr/branches/branch-0.7/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java
Fri Mar 9 16:43:33 2012
@@ -29,6 +29,8 @@ import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import java.io.IOException;
+import java.math.BigInteger;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -37,12 +39,13 @@ import org.apache.whirr.Cluster;
import org.apache.whirr.Cluster.Instance;
import org.apache.whirr.service.ClusterActionEvent;
import org.apache.whirr.service.ClusterActionHandlerSupport;
+import org.apache.whirr.service.jclouds.StatementBuilder;
public class CassandraClusterActionHandler extends ClusterActionHandlerSupport
{
public static final String CASSANDRA_ROLE = "cassandra";
public static final int CLIENT_PORT = 9160;
- public static final int JMX_PORT = 8080;
+ public static final int JMX_PORT = 7199;
public static final String BIN_TARBALL = "whirr.cassandra.tarball.url";
public static final String MAJOR_VERSION = "whirr.cassandra.version.major";
@@ -51,7 +54,7 @@ public class CassandraClusterActionHandl
public String getRole() {
return CASSANDRA_ROLE;
}
-
+
@Override
protected void beforeBootstrap(ClusterActionEvent event) throws IOException {
ClusterSpec clusterSpec = event.getClusterSpec();
@@ -76,39 +79,69 @@ public class CassandraClusterActionHandl
}
@Override
- protected void beforeConfigure(ClusterActionEvent event)
- throws IOException, InterruptedException {
+ protected void beforeConfigure(final ClusterActionEvent event)
+ throws IOException, InterruptedException {
Cluster cluster = event.getCluster();
+ Set<Instance> instances =
cluster.getInstancesMatching(role(CASSANDRA_ROLE));
event.getFirewallManager().addRule(
Rule.create()
- .destination(cluster.getInstancesMatching(role(CASSANDRA_ROLE)))
+ .destination(instances)
.ports(CLIENT_PORT, JMX_PORT)
);
- List<Instance> seeds = getSeeds(cluster.getInstances());
- String servers = Joiner.on(' ').join(getPrivateIps(seeds));
- addStatement(event, call("configure_cassandra", servers));
+ setInitialTokensAsEnvironmentVariables(event, instances);
+
+ List<Instance> seeds = getSeeds(instances);
+ String seedServers = Joiner.on(' ').join(getPrivateIps(seeds));
+
+ addStatement(event, call("configure_cassandra", seedServers));
addStatement(event, call("start_cassandra"));
}
private List<String> getPrivateIps(List<Instance> instances) {
return Lists.transform(Lists.newArrayList(instances),
- new Function<Instance, String>() {
- @Override
- public String apply(Instance instance) {
- return instance.getPrivateIp();
- }
- });
+ new Function<Instance, String>() {
+ @Override
+ public String apply(Instance instance) {
+ return instance.getPrivateIp();
+ }
+ });
}
/**
+ * Compute initial_token for a balanced cluster
+ */
+ protected List<String> computeInitialTokens(int numberOfNodes) {
+ List<String> tokens = Lists.newArrayList();
+
+ BigInteger step = new BigInteger("2")
+ .pow(127).divide(BigInteger.valueOf(numberOfNodes));
+
+ for (int i = 0; i < numberOfNodes; i++) {
+ tokens.add(step.multiply(BigInteger.valueOf(i)).toString());
+ }
+
+ return tokens;
+ }
+
+ private void setInitialTokensAsEnvironmentVariables(ClusterActionEvent
event, Set<Instance> instances) {
+ List<String> tokens = computeInitialTokens(instances.size());
+
+ StatementBuilder statementBuilder = event.getStatementBuilder();
+ Iterator it = tokens.iterator();
+
+ for (Instance instance : instances) {
+ statementBuilder.addExportPerInstance(instance.getId(),
"cassandraInitialToken", (String) it.next());
+ }
+ }
+
+ /**
* Pick a selection of the nodes that are to become seeds. TODO improve
* selection method. Right now it picks 20% of the nodes as seeds, or a
* minimum of one node if it is a small cluster.
- *
- * @param instances
- * all nodes in cluster
+ *
+ * @param instances all nodes in cluster
* @return list of seeds
*/
protected List<Instance> getSeeds(Set<Instance> instances) {
Modified:
whirr/branches/branch-0.7/services/cassandra/src/main/resources/functions/configure_cassandra.sh
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cassandra/src/main/resources/functions/configure_cassandra.sh?rev=1298911&r1=1298910&r2=1298911&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/cassandra/src/main/resources/functions/configure_cassandra.sh
(original)
+++
whirr/branches/branch-0.7/services/cassandra/src/main/resources/functions/configure_cassandra.sh
Fri Mar 9 16:43:33 2012
@@ -52,6 +52,8 @@ function configure_cassandra() {
sed -i -e "s|listen_address: localhost|listen_address: $PRIVATE_IP|"
$config_file
sed -i -e "s|rpc_address: localhost|rpc_address: 0.0.0.0|" $config_file
+ sed -i -e "s|cluster_name: 'Test Cluster'|cluster_name: 'Whirr'|"
$config_file
+ sed -i -e "s|initial_token:.*|initial_token: $CASSANDRA_INITIAL_TOKEN|"
$config_file
fi
}
Modified:
whirr/branches/branch-0.7/services/cassandra/src/main/resources/functions/install_cassandra.sh
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cassandra/src/main/resources/functions/install_cassandra.sh?rev=1298911&r1=1298910&r2=1298911&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/cassandra/src/main/resources/functions/install_cassandra.sh
(original)
+++
whirr/branches/branch-0.7/services/cassandra/src/main/resources/functions/install_cassandra.sh
Fri Mar 9 16:43:33 2012
@@ -16,9 +16,9 @@
#
function install_cassandra() {
- C_MAJOR_VERSION=${1:-0.7}
-
C_TAR_URL=${2:-http://archive.apache.org/dist/cassandra/0.7.5/apache-cassandra-0.7.5-bin.tar.gz}
-
+ C_MAJOR_VERSION=${1:-1.0}
+
C_TAR_URL=${2:-http://archive.apache.org/dist/cassandra/1.0.6/apache-cassandra-1.0.6-bin.tar.gz}
+
c_tar_file=`basename $C_TAR_URL`
c_tar_dir=`echo $c_tar_file | awk -F '-bin' '{print $1}'`
Modified:
whirr/branches/branch-0.7/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandlerTest.java
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandlerTest.java?rev=1298911&r1=1298910&r2=1298911&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandlerTest.java
(original)
+++
whirr/branches/branch-0.7/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandlerTest.java
Fri Mar 9 16:43:33 2012
@@ -20,6 +20,7 @@ package org.apache.whirr.service.cassand
import static org.junit.Assert.assertEquals;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.net.UnknownHostException;
@@ -62,4 +63,17 @@ public class CassandraClusterActionHandl
assertEquals(one, seeds2.get(0));
assertEquals(two, seeds2.get(1));
}
+
+ @Test
+ public void testComputeInitialTokens() {
+ List<String> result = new
CassandraClusterActionHandler().computeInitialTokens(5);
+
+ assertEquals(result, ImmutableList.of(
+ "0",
+ "34028236692093846346337460743176821145",
+ "68056473384187692692674921486353642290",
+ "102084710076281539039012382229530463435",
+ "136112946768375385385349842972707284580"
+ ));
+ }
}
Modified:
whirr/branches/branch-0.7/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/integration/CassandraServiceTest.java
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/integration/CassandraServiceTest.java?rev=1298911&r1=1298910&r2=1298911&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/integration/CassandraServiceTest.java
(original)
+++
whirr/branches/branch-0.7/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/integration/CassandraServiceTest.java
Fri Mar 9 16:43:33 2012
@@ -82,7 +82,7 @@ public class CassandraServiceTest {
LOG.info("Waiting for Cassandra to start");
for (Instance instance : cluster.getInstances()) {
int tries = 0;
- while (tries < 10) {
+ while (tries < 30) {
try {
Cassandra.Client client = client(instance);
client.describe_cluster_name();
@@ -93,7 +93,7 @@ public class CassandraServiceTest {
} catch (TException e) {
try {
LOG.warn(instance.getPublicIp() + " not reachable, try #" + tries
+ ", waiting 1s");
- Thread.sleep(1000);
+ Thread.sleep(10000);
} catch (InterruptedException e1) {
break;
}
Modified:
whirr/branches/branch-0.7/services/cassandra/src/test/resources/whirr-cassandra-test.properties
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cassandra/src/test/resources/whirr-cassandra-test.properties?rev=1298911&r1=1298910&r2=1298911&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/cassandra/src/test/resources/whirr-cassandra-test.properties
(original)
+++
whirr/branches/branch-0.7/services/cassandra/src/test/resources/whirr-cassandra-test.properties
Fri Mar 9 16:43:33 2012
@@ -22,6 +22,6 @@ whirr.provider=${sys:whirr.test.provider
whirr.identity=${sys:whirr.test.identity}
whirr.credential=${sys:whirr.test.credential}
-# Waiting for http://code.google.com/p/jclouds/issues/detail?id=746
+# Waiting for http://code.google.com/p/jclouds/issues/detail?id=746
# whirr.hardware-min-ram=512
