Author: asavu
Date: Fri Nov 18 21:47:03 2011
New Revision: 1203864
URL: http://svn.apache.org/viewvc?rev=1203864&view=rev
Log:
WHIRR-415. Let users specify the CDH release (cdh3u1, cdh3u2) (Alex Heneveld
and asavu)
Modified:
whirr/trunk/CHANGES.txt
whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/StatementBuilderTest.java
whirr/trunk/services/cdh/pom.xml
whirr/trunk/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHadoopServiceTest.java
whirr/trunk/services/cdh/src/test/resources/whirr-hadoop-test.properties
whirr/trunk/services/cdh/src/test/resources/whirr-hbase-test.properties
whirr/trunk/services/cdh/src/test/resources/whirr-zookeeper-test.properties
Modified: whirr/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/whirr/trunk/CHANGES.txt?rev=1203864&r1=1203863&r2=1203864&view=diff
==============================================================================
--- whirr/trunk/CHANGES.txt (original)
+++ whirr/trunk/CHANGES.txt Fri Nov 18 21:47:03 2011
@@ -82,6 +82,9 @@ Trunk (unreleased changes)
WHIRR-416. Enable lazy image fetching when the image-id is known (asavu)
+ WHIRR-415. Let users specify the CDH release (cdh3u1, cdh3u2)
+ (Alex Heneveld and asavu)
+
BUG FIXES
WHIRR-377. Fix broken CLI logging config. (asavu via tomwhite)
Modified:
whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java?rev=1203864&r1=1203863&r2=1203864&view=diff
==============================================================================
---
whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
(original)
+++
whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
Fri Nov 18 21:47:03 2011
@@ -33,6 +33,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -65,7 +66,7 @@ public class StatementBuilder {
@Override
public String render(OsFamily family) {
ScriptBuilder scriptBuilder = new ScriptBuilder();
- Map<String, String> metadataMap = Maps.newHashMap();
+ Map<String, String> metadataMap = Maps.newLinkedHashMap();
metadataMap.putAll(
ImmutableMap.of(
"clusterName", clusterSpec.getClusterName(),
@@ -91,6 +92,13 @@ public class StatementBuilder {
LOG.warn("Could not resolve hostname for " + instance, e);
}
}
+ for (Iterator<?> it =
clusterSpec.getConfiguration().getKeys("whirr.env"); it.hasNext(); ) {
+ String key = (String)it.next();
+ String value = clusterSpec.getConfiguration().getString(key);
+ metadataMap.put(key.substring("whirr.env.".length()), value);
+ }
+ metadataMap.putAll(exports);
+
// Write export statements out directly
// Using InitBuilder would be a possible improvement
String writeVariableExporters =
Utils.writeVariableExporters(metadataMap, family);
@@ -107,6 +115,7 @@ public class StatementBuilder {
}
protected List<Statement> statements = Lists.newArrayList();
+ protected Map<String,String> exports = Maps.newLinkedHashMap();
public void addStatement(Statement statement) {
if (!statements.contains(statement)) {
@@ -119,6 +128,10 @@ public class StatementBuilder {
addStatement(statement);
}
}
+
+ public void addExport(String key, String value) {
+ exports.put(key, value);
+ }
public Statement build(ClusterSpec clusterSpec) {
return build(clusterSpec, null);
Modified:
whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/StatementBuilderTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/StatementBuilderTest.java?rev=1203864&r1=1203863&r2=1203864&view=diff
==============================================================================
---
whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/StatementBuilderTest.java
(original)
+++
whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/StatementBuilderTest.java
Fri Nov 18 21:47:03 2011
@@ -21,10 +21,13 @@ package org.apache.whirr.service.jclouds
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertThat;
+import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.whirr.ClusterSpec;
import org.jclouds.scriptbuilder.domain.OsFamily;
+import org.jclouds.scriptbuilder.domain.Statements;
import org.junit.Test;
public class StatementBuilderTest {
@@ -54,4 +57,45 @@ public class StatementBuilderTest {
assertThat(script, containsString("runurl http://example.org/d/e f"));
}
+ @Test
+ public void testExplicitExports() throws Exception {
+ ClusterSpec clusterSpec = ClusterSpec.withTemporaryKeys();
+
+ clusterSpec.setClusterName("test-cluster");
+ clusterSpec.setProvider("test-provider");
+
+ StatementBuilder builder = new StatementBuilder();
+
+ builder.addExport("fooBar", "my_value");
+ builder.addStatement(Statements.exec("echo $FOO_BAR"));
+
+ String script = builder.build(clusterSpec).render(OsFamily.UNIX);
+
+ assertThat(script, containsString("export FOO_BAR="));
+ assertThat(script, containsString("my_value"));
+ assertThat(script, containsString("echo $FOO_BAR"));
+ }
+
+ @Test
+ public void testClusterSpecExports() throws Exception {
+ PropertiesConfiguration conf = new PropertiesConfiguration();
+ conf.addProperty("ignored", "ignored_value");
+ conf.addProperty("whirr.env.fooBar", "my_value");
+
+ ClusterSpec clusterSpec = ClusterSpec.withTemporaryKeys(conf);
+ clusterSpec.setClusterName("test-cluster");
+ clusterSpec.setProvider("test-provider");
+
+ StatementBuilder builder = new StatementBuilder();
+
+ builder.addStatement(Statements.exec("echo $FOO_BAR"));
+
+ String script = builder.build(clusterSpec).render(OsFamily.UNIX);
+
+ assertThat(script, containsString("export FOO_BAR="));
+ assertThat(script, containsString("my_value"));
+ assertThat(script, containsString("echo $FOO_BAR"));
+ assertThat(script, not(containsString("ignored")));
+ }
+
}
Modified: whirr/trunk/services/cdh/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/cdh/pom.xml?rev=1203864&r1=1203863&r2=1203864&view=diff
==============================================================================
--- whirr/trunk/services/cdh/pom.xml (original)
+++ whirr/trunk/services/cdh/pom.xml Fri Nov 18 21:47:03 2011
@@ -98,21 +98,29 @@
<artifactId>dnsjava</artifactId>
</dependency>
<dependency>
+ <!-- look at
https://repository.cloudera.com/artifactory/libs-release-local/org/apache/hadoop/hadoop-core/
+ to find version info for other cdh REPO versions, same for hbase
and zookeeper -->
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
- <version>0.20.2-cdh3u0</version>
+ <!-- <version>0.20.2-cdh3u0</version> -->
+ <!-- <version>0.20.2-cdh3u1</version> -->
+ <version>0.20.2-cdh3u2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
- <version>0.90.1-cdh3u0</version>
+ <!-- <version>0.90.1-cdh3u0</version> -->
+ <!-- <version>0.90.3-cdh3u1</version> -->
+ <version>0.90.4-cdh3u2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
- <version>3.3.3-cdh3u0</version>
+ <!-- <version>3.3.3-cdh3u0</version> -->
+ <!-- <version>3.3.3-cdh3u1</version> -->
+ <version>3.3.3-cdh3u2</version>
<scope>test</scope>
</dependency>
<!-- The following are missing CDH dependencies -->
Modified:
whirr/trunk/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHadoopServiceTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHadoopServiceTest.java?rev=1203864&r1=1203863&r2=1203864&view=diff
==============================================================================
---
whirr/trunk/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHadoopServiceTest.java
(original)
+++
whirr/trunk/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHadoopServiceTest.java
Fri Nov 18 21:47:03 2011
@@ -20,6 +20,7 @@ package org.apache.whirr.service.cdh.int
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.failNotEquals;
import java.io.BufferedReader;
import java.io.IOException;
@@ -27,8 +28,11 @@ import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.util.Map;
import java.util.Map.Entry;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.conf.Configuration;
@@ -48,24 +52,39 @@ import org.apache.whirr.Cluster;
import org.apache.whirr.ClusterController;
import org.apache.whirr.ClusterSpec;
import org.apache.whirr.service.hadoop.HadoopProxy;
-import org.junit.After;
-import org.junit.Before;
+import org.jclouds.compute.domain.ExecResponse;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.Statements;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class CdhHadoopServiceTest {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(CdhHadoopServiceTest.class);
+
+ private final static Predicate<NodeMetadata> ALL = Predicates.alwaysTrue();
+
+ protected static ClusterSpec clusterSpec;
+ protected static ClusterController controller;
+ protected static HadoopProxy proxy;
+ protected static Cluster cluster;
+
+ protected static String getPropertiesFilename() {
+ return "whirr-hadoop-test.properties";
+ }
- private ClusterSpec clusterSpec;
- private ClusterController controller;
- private HadoopProxy proxy;
- private Cluster cluster;
-
- @Before
- public void setUp() throws Exception {
+ @BeforeClass
+ public static void setUp() throws Exception {
CompositeConfiguration config = new CompositeConfiguration();
if (System.getProperty("config") != null) {
config.addConfiguration(new
PropertiesConfiguration(System.getProperty("config")));
}
- config.addConfiguration(new
PropertiesConfiguration("whirr-hadoop-test.properties"));
+ config.addConfiguration(new
PropertiesConfiguration(getPropertiesFilename()));
clusterSpec = ClusterSpec.withTemporaryKeys(config);
controller = new ClusterController();
@@ -73,15 +92,37 @@ public class CdhHadoopServiceTest {
proxy = new HadoopProxy(clusterSpec, cluster);
proxy.start();
}
-
+
+ @AfterClass
+ public static void tearDown() throws IOException, InterruptedException {
+ if (proxy != null) {
+ proxy.stop();
+ }
+ controller.destroyCluster(clusterSpec);
+ }
+
+ @Test
+ public void testVersion() throws Exception {
+ Statement checkVersion = Statements.exec("ls
/etc/alternatives/hadoop-lib");
+ Map<? extends NodeMetadata, ExecResponse> responses =
+ controller.runScriptOnNodesMatching(clusterSpec, ALL, checkVersion);
+
+ printResponses(checkVersion, responses);
+ assertResponsesContain(responses, checkVersion, "cdh3u2");
+ }
+
@Test
- public void test() throws Exception {
+ public void testJobExecution() throws Exception {
Configuration conf = getConfiguration();
JobConf job = new JobConf(conf, CdhHadoopServiceTest.class);
JobClient client = new JobClient(job);
waitForTaskTrackers(client);
+ checkHadoop(conf, client, job);
+ }
+
+ protected void checkHadoop(Configuration conf, JobClient client, JobConf
job) throws Exception {
FileSystem fs = FileSystem.get(conf);
OutputStream os = fs.create(new Path("input"));
@@ -104,7 +145,6 @@ public class CdhHadoopServiceTest {
assertEquals("b\t1", reader.readLine());
assertNull(reader.readLine());
reader.close();
-
}
private Configuration getConfiguration() {
@@ -130,13 +170,27 @@ public class CdhHadoopServiceTest {
}
}
}
-
- @After
- public void tearDown() throws IOException, InterruptedException {
- if (proxy != null) {
- proxy.stop();
+
+ private static void assertResponsesContain(
+ Map<? extends NodeMetadata, ExecResponse> responses, Statement statement,
+ String text) {
+ for (Map.Entry<? extends NodeMetadata, ExecResponse> entry : responses
+ .entrySet()) {
+ if (!entry.getValue().getOutput().contains(text)) {
+ failNotEquals("Node: " + entry.getKey().getId()
+ + " failed to execute the command: " + statement
+ + " as could not find expected text", text, entry.getValue());
+ }
+ }
+ }
+
+ public static void printResponses(Statement statement,
+ Map<? extends NodeMetadata, ExecResponse> responses) {
+ LOG.info("Responses for Statement: " + statement);
+ for (Map.Entry<? extends NodeMetadata, ExecResponse> entry : responses
+ .entrySet()) {
+ LOG.info("Node[" + entry.getKey().getId() + "]: " + entry.getValue());
}
- controller.destroyCluster(clusterSpec);
}
}
Modified:
whirr/trunk/services/cdh/src/test/resources/whirr-hadoop-test.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/cdh/src/test/resources/whirr-hadoop-test.properties?rev=1203864&r1=1203863&r2=1203864&view=diff
==============================================================================
--- whirr/trunk/services/cdh/src/test/resources/whirr-hadoop-test.properties
(original)
+++ whirr/trunk/services/cdh/src/test/resources/whirr-hadoop-test.properties
Fri Nov 18 21:47:03 2011
@@ -17,12 +17,14 @@
#
whirr.cluster-name=hadoop-cdh-itest-${sys:whirr.test.provider}-${sys:user.name}
-whirr.instance-templates=1 hadoop-jobtracker+hadoop-namenode,1
hadoop-datanode+hadoop-tasktracker
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker,1
hadoop-datanode+hadoop-tasktracker
whirr.provider=${sys:whirr.test.provider}
whirr.identity=${sys:whirr.test.identity}
whirr.credential=${sys:whirr.test.credential}
+whirr.env.repo=cdh3u2
+
whirr.hadoop.install-function=install_cdh_hadoop
whirr.hadoop.configure-function=configure_cdh_hadoop
Modified:
whirr/trunk/services/cdh/src/test/resources/whirr-hbase-test.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/cdh/src/test/resources/whirr-hbase-test.properties?rev=1203864&r1=1203863&r2=1203864&view=diff
==============================================================================
--- whirr/trunk/services/cdh/src/test/resources/whirr-hbase-test.properties
(original)
+++ whirr/trunk/services/cdh/src/test/resources/whirr-hbase-test.properties Fri
Nov 18 21:47:03 2011
@@ -23,6 +23,8 @@ whirr.provider=${sys:whirr.test.provider
whirr.identity=${sys:whirr.test.identity}
whirr.credential=${sys:whirr.test.credential}
+whirr.env.repo=cdh3u2
+
whirr.hadoop.install-function=install_cdh_hadoop
whirr.hadoop.configure-function=configure_cdh_hadoop
Modified:
whirr/trunk/services/cdh/src/test/resources/whirr-zookeeper-test.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/cdh/src/test/resources/whirr-zookeeper-test.properties?rev=1203864&r1=1203863&r2=1203864&view=diff
==============================================================================
--- whirr/trunk/services/cdh/src/test/resources/whirr-zookeeper-test.properties
(original)
+++ whirr/trunk/services/cdh/src/test/resources/whirr-zookeeper-test.properties
Fri Nov 18 21:47:03 2011
@@ -22,6 +22,8 @@ whirr.provider=${sys:whirr.test.provider
whirr.identity=${sys:whirr.test.identity}
whirr.credential=${sys:whirr.test.credential}
+whirr.env.repo=cdh3u2
+
whirr.zookeeper.install-function=install_cdh_zookeeper
whirr.zookeeper.configure-function=configure_cdh_zookeeper
