Author: asavu
Date: Sat Dec 10 23:23:33 2011
New Revision: 1212927
URL: http://svn.apache.org/viewvc?rev=1212927&view=rev
Log:
WHIRR-384. Add Mahout as a service (Frank Scholten via asavu)
Added:
whirr/trunk/recipes/mahout-ec2.properties
whirr/trunk/services/mahout/ (with props)
whirr/trunk/services/mahout/pom.xml
whirr/trunk/services/mahout/src/
whirr/trunk/services/mahout/src/main/
whirr/trunk/services/mahout/src/main/java/
whirr/trunk/services/mahout/src/main/java/org/
whirr/trunk/services/mahout/src/main/java/org/apache/
whirr/trunk/services/mahout/src/main/java/org/apache/whirr/
whirr/trunk/services/mahout/src/main/java/org/apache/whirr/service/
whirr/trunk/services/mahout/src/main/java/org/apache/whirr/service/mahout/
whirr/trunk/services/mahout/src/main/java/org/apache/whirr/service/mahout/MahoutClientClusterActionHandler.java
whirr/trunk/services/mahout/src/main/resources/
whirr/trunk/services/mahout/src/main/resources/META-INF/
whirr/trunk/services/mahout/src/main/resources/META-INF/services/
whirr/trunk/services/mahout/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
whirr/trunk/services/mahout/src/main/resources/functions/
whirr/trunk/services/mahout/src/main/resources/functions/configure_mahout_client.sh
whirr/trunk/services/mahout/src/main/resources/whirr-mahout-default.properties
whirr/trunk/services/mahout/src/test/
whirr/trunk/services/mahout/src/test/java/
whirr/trunk/services/mahout/src/test/java/org/
whirr/trunk/services/mahout/src/test/java/org/apache/
whirr/trunk/services/mahout/src/test/java/org/apache/whirr/
whirr/trunk/services/mahout/src/test/java/org/apache/whirr/service/
whirr/trunk/services/mahout/src/test/java/org/apache/whirr/service/mahout/
whirr/trunk/services/mahout/src/test/java/org/apache/whirr/service/mahout/integration/
whirr/trunk/services/mahout/src/test/java/org/apache/whirr/service/mahout/integration/MahoutServiceTest.java
whirr/trunk/services/mahout/src/test/resources/
whirr/trunk/services/mahout/src/test/resources/log4j.xml
whirr/trunk/services/mahout/src/test/resources/whirr-mahout-test.properties
Modified:
whirr/trunk/CHANGES.txt
whirr/trunk/cli/pom.xml
whirr/trunk/core/src/main/resources/functions/install_java.sh
whirr/trunk/pom.xml
whirr/trunk/services/hadoop/src/main/resources/functions/install_hadoop.sh
Modified: whirr/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/whirr/trunk/CHANGES.txt?rev=1212927&r1=1212926&r2=1212927&view=diff
==============================================================================
--- whirr/trunk/CHANGES.txt (original)
+++ whirr/trunk/CHANGES.txt Sat Dec 10 23:23:33 2011
@@ -10,6 +10,8 @@ Trunk (unreleased changes)
WHIRR-49. Allow Whirr to use Chef for configuration management
+ WHIRR-384. Add Mahout as a service (Frank Scholten via asavu)
+
IMPROVEMENTS
WHIRR-116. Site should have docs for each released version. (tomwhite)
Modified: whirr/trunk/cli/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/cli/pom.xml?rev=1212927&r1=1212926&r2=1212927&view=diff
==============================================================================
--- whirr/trunk/cli/pom.xml (original)
+++ whirr/trunk/cli/pom.xml Sat Dec 10 23:23:33 2011
@@ -56,6 +56,11 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>whirr-mahout</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>whirr-zookeeper</artifactId>
<version>${project.version}</version>
</dependency>
Modified: whirr/trunk/core/src/main/resources/functions/install_java.sh
URL:
http://svn.apache.org/viewvc/whirr/trunk/core/src/main/resources/functions/install_java.sh?rev=1212927&r1=1212926&r2=1212927&view=diff
==============================================================================
--- whirr/trunk/core/src/main/resources/functions/install_java.sh (original)
+++ whirr/trunk/core/src/main/resources/functions/install_java.sh Sat Dec 10
23:23:33 2011
@@ -40,6 +40,7 @@ sun-java6-jre shared/present-sun-dlj-v
apt-get -y install sun-java6-jdk
echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> /etc/profile
+ echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> ~root/.bashrc
export JAVA_HOME=/usr/lib/jvm/java-6-sun
java -version
Modified: whirr/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/pom.xml?rev=1212927&r1=1212926&r2=1212927&view=diff
==============================================================================
--- whirr/trunk/pom.xml (original)
+++ whirr/trunk/pom.xml Sat Dec 10 23:23:33 2011
@@ -50,6 +50,7 @@
<module>services/hama</module>
<module>services/puppet</module>
<module>services/chef</module>
+ <module>services/mahout</module>
</modules>
<properties>
Added: whirr/trunk/recipes/mahout-ec2.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/recipes/mahout-ec2.properties?rev=1212927&view=auto
==============================================================================
--- whirr/trunk/recipes/mahout-ec2.properties (added)
+++ whirr/trunk/recipes/mahout-ec2.properties Sat Dec 10 23:23:33 2011
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Read the Configuration Guide for more info:
+# http://incubator.apache.org/whirr/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=mahout
+
+whirr.instance-templates=1 hadoop-jobtracker+hadoop-namenode+mahout-client,1
hadoop-datanode+hadoop-tasktracker
+
+whirr.mahout.version=0.5
+whirr.mahout.tarball.url=http://archive.apache.org/dist/mahout/${whirr.mahout.version}/mahout-distribution-${whirr.mahout.version}.tar.gz
+
+# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment
variables.
+whirr.provider=aws-ec2
+whirr.identity=${env:AWS_ACCESS_KEY_ID}
+whirr.credential=${env:AWS_SECRET_ACCESS_KEY}
+
+# By default use the user system SSH keys. Override them here.
+# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
+# whirr.public-key-file=${whirr.private-key-file}.pub
+
+
Modified:
whirr/trunk/services/hadoop/src/main/resources/functions/install_hadoop.sh
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/resources/functions/install_hadoop.sh?rev=1212927&r1=1212926&r2=1212927&view=diff
==============================================================================
--- whirr/trunk/services/hadoop/src/main/resources/functions/install_hadoop.sh
(original)
+++ whirr/trunk/services/hadoop/src/main/resources/functions/install_hadoop.sh
Sat Dec 10 23:23:33 2011
@@ -53,7 +53,10 @@ function install_hadoop() {
echo "export HADOOP_HOME=$HADOOP_HOME" >> ~root/.bashrc
echo 'export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$PATH' >> ~root/.bashrc
-
+
+ echo "export HADOOP_HOME=$HADOOP_HOME" >> /etc/profile
+ echo 'export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$PATH' >> /etc/profile
+
INSTALL_HADOOP_DONE=1
}
Propchange: whirr/trunk/services/mahout/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sat Dec 10 23:23:33 2011
@@ -0,0 +1 @@
+target
Added: whirr/trunk/services/mahout/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/mahout/pom.xml?rev=1212927&view=auto
==============================================================================
--- whirr/trunk/services/mahout/pom.xml (added)
+++ whirr/trunk/services/mahout/pom.xml Sat Dec 10 23:23:33 2011
@@ -0,0 +1,64 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr</artifactId>
+ <version>0.7.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-mahout</artifactId>
+ <packaging>jar</packaging>
+ <version>0.7.0-SNAPSHOT</version>
+ <name>Apache Whirr Mahout</name>
+ <properties>
+ <hadoop.version>0.20.2</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
Added:
whirr/trunk/services/mahout/src/main/java/org/apache/whirr/service/mahout/MahoutClientClusterActionHandler.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/mahout/src/main/java/org/apache/whirr/service/mahout/MahoutClientClusterActionHandler.java?rev=1212927&view=auto
==============================================================================
---
whirr/trunk/services/mahout/src/main/java/org/apache/whirr/service/mahout/MahoutClientClusterActionHandler.java
(added)
+++
whirr/trunk/services/mahout/src/main/java/org/apache/whirr/service/mahout/MahoutClientClusterActionHandler.java
Sat Dec 10 23:23:33 2011
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.mahout;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.ClusterActionHandlerSupport;
+
+import java.io.IOException;
+
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+/**
+ * Mahout cluster action handler which configures Mahout by unpacking a binary
tarball and
+ * setting MAHOUT_HOME and PATH environment variables.
+ */
+public class MahoutClientClusterActionHandler extends
ClusterActionHandlerSupport {
+
+ public final static String MAHOUT_CLIENT_ROLE = "mahout-client";
+
+ final static String MAHOUT_DEFAULT_PROPERTIES =
"whirr-mahout-default.properties";
+
+ final static String MAHOUT_TAR_URL = "whirr.mahout.tarball.url";
+
+ final static String MAHOUT_CLIENT_SCRIPT = "configure_mahout_client";
+
+ final static String URL_FLAG = "-u";
+
+ @Override
+ public String getRole() {
+ return MAHOUT_CLIENT_ROLE;
+ }
+
+ @Override
+ protected void beforeBootstrap(ClusterActionEvent event) throws IOException,
InterruptedException {
+ Configuration conf = getConfiguration(event.getClusterSpec(),
MAHOUT_DEFAULT_PROPERTIES);
+
+ String mahoutTarball = prepareRemoteFileUrl(event,
conf.getString(MAHOUT_TAR_URL));
+
+ addStatement(event, call(MAHOUT_CLIENT_SCRIPT, URL_FLAG, mahoutTarball));
+ }
+}
Added:
whirr/trunk/services/mahout/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/mahout/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler?rev=1212927&view=auto
==============================================================================
---
whirr/trunk/services/mahout/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
(added)
+++
whirr/trunk/services/mahout/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
Sat Dec 10 23:23:33 2011
@@ -0,0 +1,13 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.whirr.service.mahout.MahoutClientClusterActionHandler
+
Added:
whirr/trunk/services/mahout/src/main/resources/functions/configure_mahout_client.sh
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/mahout/src/main/resources/functions/configure_mahout_client.sh?rev=1212927&view=auto
==============================================================================
---
whirr/trunk/services/mahout/src/main/resources/functions/configure_mahout_client.sh
(added)
+++
whirr/trunk/services/mahout/src/main/resources/functions/configure_mahout_client.sh
Sat Dec 10 23:23:33 2011
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function configure_mahout_client() {
+ local OPTARG
+
+ MAHOUT_TAR_URL=
+ while getopts "u:" OPTION; do
+ case $OPTION in
+ u)
+ MAHOUT_TAR_URL="$OPTARG"
+ ;;
+ esac
+ done
+
+ MAHOUT_HOME=/usr/local/$(basename $MAHOUT_TAR_URL .tar.gz)
+
+ install_tarball $MAHOUT_TAR_URL
+ ln -s $MAHOUT_HOME /usr/local/mahout
+
+ echo "export MAHOUT_HOME=$MAHOUT_HOME" >> /etc/profile
+ echo 'export PATH=$MAHOUT_HOME/bin:$PATH' >> /etc/profile
+}
\ No newline at end of file
Added:
whirr/trunk/services/mahout/src/main/resources/whirr-mahout-default.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/mahout/src/main/resources/whirr-mahout-default.properties?rev=1212927&view=auto
==============================================================================
---
whirr/trunk/services/mahout/src/main/resources/whirr-mahout-default.properties
(added)
+++
whirr/trunk/services/mahout/src/main/resources/whirr-mahout-default.properties
Sat Dec 10 23:23:33 2011
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+whirr.mahout.version=0.5
+
+whirr.mahout.tarball.url=http://archive.apache.org/dist/mahout/${whirr.mahout.version}/mahout-distribution-${whirr.mahout.version}.tar.gz
\ No newline at end of file
Added:
whirr/trunk/services/mahout/src/test/java/org/apache/whirr/service/mahout/integration/MahoutServiceTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/mahout/src/test/java/org/apache/whirr/service/mahout/integration/MahoutServiceTest.java?rev=1212927&view=auto
==============================================================================
---
whirr/trunk/services/mahout/src/test/java/org/apache/whirr/service/mahout/integration/MahoutServiceTest.java
(added)
+++
whirr/trunk/services/mahout/src/test/java/org/apache/whirr/service/mahout/integration/MahoutServiceTest.java
Sat Dec 10 23:23:33 2011
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.mahout.integration;
+
+import com.google.common.base.Predicate;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.state.ClusterStateStoreFactory;
+import org.jclouds.compute.domain.ExecResponse;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.Statements;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static com.google.common.base.Predicates.alwaysTrue;
+import static com.google.common.base.Predicates.and;
+import static com.google.common.collect.Sets.newHashSet;
+import static junit.framework.Assert.failNotEquals;
+import static org.apache.whirr.RolePredicates.anyRoleIn;
+import static
org.apache.whirr.service.mahout.MahoutClientClusterActionHandler.MAHOUT_CLIENT_ROLE;
+import static org.jclouds.compute.predicates.NodePredicates.withIds;
+
+/**
+ * Install the mahout binary distribution.
+ */
+public class MahoutServiceTest {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(MahoutServiceTest.class);
+
+ private static ClusterSpec clusterSpec;
+ private static ClusterController controller;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ CompositeConfiguration config = new CompositeConfiguration();
+ if (System.getProperty("config") != null) {
+ config.addConfiguration(new
PropertiesConfiguration(System.getProperty("config")));
+ }
+ config.addConfiguration(new
PropertiesConfiguration("whirr-mahout-test.properties"));
+ clusterSpec = ClusterSpec.withTemporaryKeys(config);
+ controller = new ClusterController();
+ controller.launchCluster(clusterSpec);
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException, InterruptedException {
+ if (controller != null) {
+ controller.destroyCluster(clusterSpec);
+ }
+ }
+
+ @Test
+ public void testBinMahout() throws Exception {
+ Statement binMahout = Statements.exec("source /etc/profile;
$MAHOUT_HOME/bin/mahout");
+
+ Cluster.Instance mahoutInstance = findMahoutInstance();
+ Predicate<NodeMetadata> mahoutClientRole = and(alwaysTrue(),
withIds(mahoutInstance.getId()));
+
+ Map<? extends NodeMetadata, ExecResponse> responses =
controller.runScriptOnNodesMatching(clusterSpec, mahoutClientRole, binMahout);
+
+ LOG.info("Responses for Statement: " + binMahout);
+ for (Map.Entry<? extends NodeMetadata, ExecResponse> entry :
responses.entrySet()) {
+ LOG.info("Node[" + entry.getKey().getId() + "]: " + entry.getValue());
+ }
+
+ assertResponsesContain(responses, binMahout, "Running on hadoop");
+ }
+
+ public static void assertResponsesContain(Map<? extends NodeMetadata,
ExecResponse> responses, Statement statement, String text) {
+ for (Map.Entry<? extends NodeMetadata, ExecResponse> entry :
responses.entrySet()) {
+ if (!entry.getValue().getOutput().contains(text)) {
+ failNotEquals("Node: " + entry.getKey().getId()
+ + " failed to execute the command: " + statement
+ + " as could not find expected text", text,
entry.getValue());
+ }
+ }
+ }
+
+ private Cluster.Instance findMahoutInstance() throws IOException {
+ Cluster cluster = new
ClusterStateStoreFactory().create(clusterSpec).load();
+ return
cluster.getInstanceMatching(anyRoleIn(newHashSet(MAHOUT_CLIENT_ROLE)));
+ }
+}
Added: whirr/trunk/services/mahout/src/test/resources/log4j.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/mahout/src/test/resources/log4j.xml?rev=1212927&view=auto
==============================================================================
--- whirr/trunk/services/mahout/src/test/resources/log4j.xml (added)
+++ whirr/trunk/services/mahout/src/test/resources/log4j.xml Sat Dec 10
23:23:33 2011
@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+ <!--
+ For more configuration infromation and examples see the Apache
+ Log4j website: http://logging.apache.org/log4j/
+ -->
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";
+ debug="false">
+
+ <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+ <param name="Threshold" value="INFO" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WIREFILE"
class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-wire.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="BLOBSTOREFILE"
class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-blobstore.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+
+ <!-- A time/date based rolling appender -->
+ <appender name="COMPUTEFILE"
class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-compute.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WHIRRFILE"
class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/whirr.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="SSHFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-ssh.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+ <appender name="ASYNCCOMPUTE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="COMPUTEFILE" />
+ </appender>
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="FILE" />
+ </appender>
+
+ <appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="WIREFILE" />
+ </appender>
+
+ <appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="BLOBSTOREFILE" />
+ </appender>
+
+ <appender name="ASYNCSSH" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="SSHFILE" />
+ </appender>
+ <!-- ================ -->
+ <!-- Limit categories -->
+ <!-- ================ -->
+ <category name="jclouds.blobstore">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCBLOBSTORE" />
+ </category>
+
+ <category name="org.jclouds">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNC" />
+ </category>
+
+ <category name="org.apache.whirr">
+ <priority value="DEBUG" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
+ <category name="jclouds.headers">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+ <category name="jclouds.compute">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCCOMPUTE" />
+ </category>
+
+ <category name="jclouds.ssh">
+ <priority value="TRACE" />
+ <appender-ref ref="ASYNCSSH" />
+ </category>
+
+ <category name="jclouds.wire">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category><!--
+
+ <category name="jclouds.signature">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+
+
+ --><!-- ======================= -->
+ <!-- Setup the Root category -->
+ <!-- ======================= -->
+
+ <root>
+ <priority value="WARN" />
+ <appender-ref ref="CONSOLE" />
+ </root>
+
+</log4j:configuration>
Added:
whirr/trunk/services/mahout/src/test/resources/whirr-mahout-test.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/mahout/src/test/resources/whirr-mahout-test.properties?rev=1212927&view=auto
==============================================================================
--- whirr/trunk/services/mahout/src/test/resources/whirr-mahout-test.properties
(added)
+++ whirr/trunk/services/mahout/src/test/resources/whirr-mahout-test.properties
Sat Dec 10 23:23:33 2011
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+whirr.cluster-name=mahout-itest-${sys:whirr.test.provider}-${sys:user.name}
+whirr.instance-templates=1 hadoop-jobtracker+hadoop-namenode+mahout-client,1
hadoop-datanode+hadoop-tasktracker
+
+whirr.mahout.version=0.5
+whirr.mahout.tarball.url=http://archive.apache.org/dist/mahout/${whirr.mahout.version}/mahout-distribution-${whirr.mahout.version}.tar.gz
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+# By default use the user system SSH keys. Override them here.
+#whirr.private-key-file=${sys:user.home}/.ssh/id_rsa_whirr
+#whirr.public-key-file=${whirr.private-key-file}.pub
