Author: asavu
Date: Thu Mar 8 08:50:59 2012
New Revision: 1298301
URL: http://svn.apache.org/viewvc?rev=1298301&view=rev
Log:
WHIRR-531. Add Pig as a service (Marco Didonna, Andrei Savu)
Added:
whirr/trunk/recipes/pig.properties
whirr/trunk/services/pig/ (with props)
whirr/trunk/services/pig/pom.xml
whirr/trunk/services/pig/src/
whirr/trunk/services/pig/src/main/
whirr/trunk/services/pig/src/main/java/
whirr/trunk/services/pig/src/main/java/org/
whirr/trunk/services/pig/src/main/java/org/apache/
whirr/trunk/services/pig/src/main/java/org/apache/whirr/
whirr/trunk/services/pig/src/main/java/org/apache/whirr/service/
whirr/trunk/services/pig/src/main/java/org/apache/whirr/service/pig/
whirr/trunk/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java
whirr/trunk/services/pig/src/main/resources/
whirr/trunk/services/pig/src/main/resources/META-INF/
whirr/trunk/services/pig/src/main/resources/META-INF/services/
whirr/trunk/services/pig/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
whirr/trunk/services/pig/src/main/resources/OSGI-INF/
whirr/trunk/services/pig/src/main/resources/OSGI-INF/blueprint/
whirr/trunk/services/pig/src/main/resources/OSGI-INF/blueprint/blueprint.xml
whirr/trunk/services/pig/src/main/resources/functions/
whirr/trunk/services/pig/src/main/resources/functions/configure_pig_client.sh
whirr/trunk/services/pig/src/main/resources/whirr-pig-default.properties
whirr/trunk/services/pig/src/test/
whirr/trunk/services/pig/src/test/java/
whirr/trunk/services/pig/src/test/java/org/
whirr/trunk/services/pig/src/test/java/org/apache/
whirr/trunk/services/pig/src/test/java/org/apache/whirr/
whirr/trunk/services/pig/src/test/java/org/apache/whirr/service/
whirr/trunk/services/pig/src/test/java/org/apache/whirr/service/pig/
whirr/trunk/services/pig/src/test/java/org/apache/whirr/service/pig/integration/
whirr/trunk/services/pig/src/test/java/org/apache/whirr/service/pig/integration/PigServiceTest.java
whirr/trunk/services/pig/src/test/resources/
whirr/trunk/services/pig/src/test/resources/log4j.xml
whirr/trunk/services/pig/src/test/resources/whirr-pig-test.properties
Modified:
whirr/trunk/CHANGES.txt
whirr/trunk/cli/pom.xml
whirr/trunk/platforms/karaf/feature/pom.xml
whirr/trunk/platforms/karaf/feature/src/main/resources/features.xml
whirr/trunk/platforms/karaf/itests/src/test/java/org/apache/whirr/karaf/itest/WhirrServicesTest.java
whirr/trunk/pom.xml
whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
Modified: whirr/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/whirr/trunk/CHANGES.txt?rev=1298301&r1=1298300&r2=1298301&view=diff
==============================================================================
--- whirr/trunk/CHANGES.txt (original)
+++ whirr/trunk/CHANGES.txt Thu Mar 8 08:50:59 2012
@@ -7,6 +7,8 @@ Trunk (unreleased changes)
WHIRR-500. Let users control which hardware is used for each
instance template (Karel Vervaeke via asavu)
+ WHIRR-531. Add Pig as a service (Marco Didonna, Andrei Savu)
+
IMPROVEMENTS
WHIRR-514. Avoid hardcoded Whirr version number in karaf tests (kve)
Modified: whirr/trunk/cli/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/cli/pom.xml?rev=1298301&r1=1298300&r2=1298301&view=diff
==============================================================================
--- whirr/trunk/cli/pom.xml (original)
+++ whirr/trunk/cli/pom.xml Thu Mar 8 08:50:59 2012
@@ -72,6 +72,11 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>whirr-pig</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>whirr-zookeeper</artifactId>
<version>${project.version}</version>
</dependency>
Modified: whirr/trunk/platforms/karaf/feature/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/platforms/karaf/feature/pom.xml?rev=1298301&r1=1298300&r2=1298301&view=diff
==============================================================================
--- whirr/trunk/platforms/karaf/feature/pom.xml (original)
+++ whirr/trunk/platforms/karaf/feature/pom.xml Thu Mar 8 08:50:59 2012
@@ -138,6 +138,7 @@
<feature>whirr-hbase</feature>
<feature>whirr-puppet</feature>
<feature>whirr-mahout</feature>
+ <feature>whirr-pig</feature>
<feature>whirr-zookeeper</feature>
</features>
<repository>target/features-repo</repository>
Modified: whirr/trunk/platforms/karaf/feature/src/main/resources/features.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/platforms/karaf/feature/src/main/resources/features.xml?rev=1298301&r1=1298300&r2=1298301&view=diff
==============================================================================
--- whirr/trunk/platforms/karaf/feature/src/main/resources/features.xml
(original)
+++ whirr/trunk/platforms/karaf/feature/src/main/resources/features.xml Thu Mar
8 08:50:59 2012
@@ -86,6 +86,10 @@
<feature version="${project.version}">whirr</feature>
<bundle>mvn:org.apache.whirr/whirr-mahout/${project.version}</bundle>
</feature>
+ <feature name="whirr-pig" version="${project.version}" description="Apache
Whirr Pig Service" resolver="(obr)">
+ <feature version="${project.version}">whirr</feature>
+ <bundle>mvn:org.apache.whirr/whirr-pig/${project.version}</bundle>
+ </feature>
<feature name="whirr-puppet" version="${project.version}"
description="Apache Whirr Puppet Service" resolver="(obr)">
<feature version="${project.version}">whirr</feature>
<bundle>mvn:org.apache.whirr/whirr-puppet/${project.version}</bundle>
Modified:
whirr/trunk/platforms/karaf/itests/src/test/java/org/apache/whirr/karaf/itest/WhirrServicesTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/platforms/karaf/itests/src/test/java/org/apache/whirr/karaf/itest/WhirrServicesTest.java?rev=1298301&r1=1298300&r2=1298301&view=diff
==============================================================================
---
whirr/trunk/platforms/karaf/itests/src/test/java/org/apache/whirr/karaf/itest/WhirrServicesTest.java
(original)
+++
whirr/trunk/platforms/karaf/itests/src/test/java/org/apache/whirr/karaf/itest/WhirrServicesTest.java
Thu Mar 8 08:50:59 2012
@@ -49,6 +49,7 @@ public class WhirrServicesTest extends W
executeCommand("features:install whirr-hbase");
executeCommand("features:install whirr-puppet");
executeCommand("features:install whirr-mahout");
+ executeCommand("features:install whirr-pig");
executeCommand("features:install whirr-zookeeper");
System.err.println(executeCommand("osgi:list"));
@@ -65,6 +66,7 @@ public class WhirrServicesTest extends W
testService("hbase-master", "hbase-regionserver", "hbase-restserver",
"hbase-avroserver", "hbase-thriftserver");
testService("puppet-install");
testService("mahout-client");
+ testService("pig-client");
testService("zookeeper");
}
Modified: whirr/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/pom.xml?rev=1298301&r1=1298300&r2=1298301&view=diff
==============================================================================
--- whirr/trunk/pom.xml (original)
+++ whirr/trunk/pom.xml Thu Mar 8 08:50:59 2012
@@ -52,6 +52,7 @@
<module>services/puppet</module>
<module>services/chef</module>
<module>services/mahout</module>
+ <module>services/pig</module>
<module>platforms/karaf</module>
</modules>
Added: whirr/trunk/recipes/pig.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/recipes/pig.properties?rev=1298301&view=auto
==============================================================================
--- whirr/trunk/recipes/pig.properties (added)
+++ whirr/trunk/recipes/pig.properties Thu Mar 8 08:50:59 2012
@@ -0,0 +1,71 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Deploy an Apache Hadoop cluster with Pig
+
+# Read the Configuration Guide for more info:
+# http://whirr.apache.org/docs/latest/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=pig
+
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+pig-client,1
hadoop-datanode+hadoop-tasktracker
+
+whirr.pig.version=0.9.2
+whirr.pig.tarball.url=http://apache.osuosl.org/pig/pig-${whirr.pig.version}/pig-${whirr.pig.version}.tar.gz
+
+whirr.hadoop.version=0.20.205.0
+whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
+
+# Setup your cloud credentials by copying conf/credentials.sample
+# to ~/.whirr/credentials and editing as needed
+
+#
+# Customise instance types & AMI for Amazon EC2
+#
+
+# Use a specific instance type. See http://aws.amazon.com/ec2/instance-types/
+# whirr.hardware-id=c1.xlarge
+
+# Ubuntu 10.04 LTS Lucid. See http://cloud.ubuntu.com/ami/
+# whirr.image-id=us-east-1/ami-35de095c
+
+# You can also specify the spot instance price:
http://aws.amazon.com/ec2/spot-instances/
+# whirr.aws-ec2-spot-price=0.15
+
+#
+# Or for Rackspace Cloud
+#
+
+# The size of the instance to use. See
http://www.rackspacecloud.com/cloud_hosting_products/servers/faq/
+# id 3: 1GB, 1 virtual core
+# id 4: 2GB, 2 virtual cores
+# id 5: 4GB, 2 virtual cores
+# id 6: 8GB, 4 virtual cores
+# id 7: 15.5GB, 4 virtual cores
+# whirr.hardware-id=6
+# Ubuntu 10.04 LTS Lucid
+# whirr.image-id=49
+
+#
+# By default use the user system SSH keys. Override them here.
+#
+
+# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
+# whirr.public-key-file=${whirr.private-key-file}.pub
+
+
Modified:
whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties?rev=1298301&r1=1298300&r2=1298301&view=diff
==============================================================================
---
whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
(original)
+++
whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
Thu Mar 8 08:50:59 2012
@@ -17,7 +17,7 @@
#
whirr.hadoop.version=0.20.205.0
-whirr.hadoop.tarball.url=http://archive.apache.org/dist/hadoop/core/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
+whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
# Hadoop defaults. The first part of the key is removed by whirr.
Propchange: whirr/trunk/services/pig/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Thu Mar 8 08:50:59 2012
@@ -0,0 +1 @@
+target
Added: whirr/trunk/services/pig/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/pig/pom.xml?rev=1298301&view=auto
==============================================================================
--- whirr/trunk/services/pig/pom.xml (added)
+++ whirr/trunk/services/pig/pom.xml Thu Mar 8 08:50:59 2012
@@ -0,0 +1,89 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr</artifactId>
+ <version>0.8.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-pig</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Whirr Pig</name>
+ <properties>
+ <osgi.import>
+ !org.apache.whirr.service.pig*,
+ org.apache.commons.configuration*;version="[1.6,2)",
+ *
+ </osgi.import>
+ <osgi.export>
+ org.apache.whirr.service.pig*;version="${project.version}"
+ </osgi.export>
+
<osgi.fragment.host>jclouds-scriptbuilder;bundle-version="${jclouds.version}"</osgi.fragment.host>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
Added:
whirr/trunk/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java?rev=1298301&view=auto
==============================================================================
---
whirr/trunk/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java
(added)
+++
whirr/trunk/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java
Thu Mar 8 08:50:59 2012
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.pig;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.ClusterActionHandlerSupport;
+
+import java.io.IOException;
+
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+/**
+ * Pig cluster action handler which configures Pig by unpacking a binary
tarball and
+ * setting PIG_HOME and PATH environment variables.
+ */
+public class PigClientClusterActionHandler extends ClusterActionHandlerSupport
{
+
+ public final static String PIG_CLIENT_ROLE = "pig-client";
+
+ final static String PIG_DEFAULT_PROPERTIES = "whirr-pig-default.properties";
+
+ final static String PIG_TAR_URL = "whirr.pig.tarball.url";
+
+ final static String PIG_CLIENT_SCRIPT = "configure_pig_client";
+
+ final static String URL_FLAG = "-u";
+
+ @Override
+ public String getRole() {
+ return PIG_CLIENT_ROLE;
+ }
+
+ @Override
+ protected void beforeBootstrap(ClusterActionEvent event) throws IOException,
InterruptedException {
+ Configuration conf = getConfiguration(event.getClusterSpec(),
PIG_DEFAULT_PROPERTIES);
+
+ String pigTarball = prepareRemoteFileUrl(event,
conf.getString(PIG_TAR_URL));
+
+ addStatement(event, call(PIG_CLIENT_SCRIPT, URL_FLAG, pigTarball));
+ }
+}
Added:
whirr/trunk/services/pig/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/pig/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler?rev=1298301&view=auto
==============================================================================
---
whirr/trunk/services/pig/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
(added)
+++
whirr/trunk/services/pig/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
Thu Mar 8 08:50:59 2012
@@ -0,0 +1,13 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.whirr.service.pig.PigClientClusterActionHandler
+
Added:
whirr/trunk/services/pig/src/main/resources/OSGI-INF/blueprint/blueprint.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/pig/src/main/resources/OSGI-INF/blueprint/blueprint.xml?rev=1298301&view=auto
==============================================================================
---
whirr/trunk/services/pig/src/main/resources/OSGI-INF/blueprint/blueprint.xml
(added)
+++
whirr/trunk/services/pig/src/main/resources/OSGI-INF/blueprint/blueprint.xml
Thu Mar 8 08:50:59 2012
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0";>
+ <service auto-export="all-classes">
+ <service-properties>
+ <entry key="name" value="pig-client"/>
+ </service-properties>
+ <bean class="org.apache.whirr.service.pig.PigClientClusterActionHandler"/>
+ </service>
+</blueprint>
Added:
whirr/trunk/services/pig/src/main/resources/functions/configure_pig_client.sh
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/pig/src/main/resources/functions/configure_pig_client.sh?rev=1298301&view=auto
==============================================================================
---
whirr/trunk/services/pig/src/main/resources/functions/configure_pig_client.sh
(added)
+++
whirr/trunk/services/pig/src/main/resources/functions/configure_pig_client.sh
Thu Mar 8 08:50:59 2012
@@ -0,0 +1,54 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function configure_pig_client() {
+
+ local OPTARG
+
+ PIG_TAR_URL=
+ while getopts "u:" OPTION; do
+ case $OPTION in
+ u)
+ PIG_TAR_URL="$OPTARG"
+ ;;
+ esac
+ done
+
+ PIG_HOME=/usr/local/$(basename $PIG_TAR_URL .tar.gz)
+
+ install_tarball $PIG_TAR_URL
+ ln -s $PIG_HOME /usr/local/pig
+
+ if [ -f /etc/profile ]; then
+ echo "export PIG_HOME=$PIG_HOME" >> /etc/profile
+ echo 'export PATH=$PIG_HOME/bin:$PATH' >> /etc/profile
+ echo 'export PIG_CLASSPATH=/etc/hadoop/conf' >> /etc/profile
+ fi
+ if [ -f /etc/bashrc ]; then
+ echo "export PIG_HOME=$PIG_HOME" >> /etc/bashrc
+ echo 'export PATH=$PIG_HOME/bin:$PATH' >> /etc/bashrc
+ echo 'export PIG_CLASSPATH=/etc/hadoop/conf' >> /etc/bashrc
+ fi
+ if [ -f /etc/skel/.bashrc ]; then
+ echo "export PIG_HOME=$PIG_HOME" >> /etc/skel/.bashrc
+ echo 'export PATH=$PIG_HOME/bin:$PATH' >> /etc/skel/.bashrc
+ echo 'export PIG_CLASSPATH=/etc/hadoop/conf' >>
/etc/skel/.bashrc
+ fi
+
+
+}
+
Added: whirr/trunk/services/pig/src/main/resources/whirr-pig-default.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/pig/src/main/resources/whirr-pig-default.properties?rev=1298301&view=auto
==============================================================================
--- whirr/trunk/services/pig/src/main/resources/whirr-pig-default.properties
(added)
+++ whirr/trunk/services/pig/src/main/resources/whirr-pig-default.properties
Thu Mar 8 08:50:59 2012
@@ -0,0 +1,14 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+whirr.pig.version=0.9.2
+whirr.pig.tarball.url=http://apache.osuosl.org/pig/pig-${whirr.pig.version}/pig-${whirr.pig.version}.tar.gz
\ No newline at end of file
Added:
whirr/trunk/services/pig/src/test/java/org/apache/whirr/service/pig/integration/PigServiceTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/pig/src/test/java/org/apache/whirr/service/pig/integration/PigServiceTest.java?rev=1298301&view=auto
==============================================================================
---
whirr/trunk/services/pig/src/test/java/org/apache/whirr/service/pig/integration/PigServiceTest.java
(added)
+++
whirr/trunk/services/pig/src/test/java/org/apache/whirr/service/pig/integration/PigServiceTest.java
Thu Mar 8 08:50:59 2012
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.pig.integration;
+
+import com.google.common.base.Predicate;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.TestConstants;
+import org.apache.whirr.state.ClusterStateStoreFactory;
+import org.jclouds.compute.domain.ExecResponse;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.Statements;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static com.google.common.base.Predicates.alwaysTrue;
+import static com.google.common.base.Predicates.and;
+import static com.google.common.collect.Sets.newHashSet;
+import static junit.framework.Assert.failNotEquals;
+import static org.apache.whirr.RolePredicates.anyRoleIn;
+import static
org.apache.whirr.service.pig.PigClientClusterActionHandler.PIG_CLIENT_ROLE;
+import static org.jclouds.compute.predicates.NodePredicates.withIds;
+
+/**
+ * Install the pig binary distribution.
+ */
+public class PigServiceTest {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(PigServiceTest.class);
+
+ private static ClusterSpec clusterSpec;
+ private static ClusterController controller;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ CompositeConfiguration config = new CompositeConfiguration();
+ if (System.getProperty("config") != null) {
+ config.addConfiguration(new
PropertiesConfiguration(System.getProperty("config")));
+ }
+ config.addConfiguration(new
PropertiesConfiguration("whirr-pig-test.properties"));
+ clusterSpec = ClusterSpec.withTemporaryKeys(config);
+ controller = new ClusterController();
+ controller.launchCluster(clusterSpec);
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException, InterruptedException {
+ if (controller != null) {
+ controller.destroyCluster(clusterSpec);
+ }
+ }
+
+ @Test(timeout = TestConstants.ITEST_TIMEOUT)
+ public void testPigBin() throws Exception {
+ Statement binPig = Statements.exec(". /etc/profile && pig -e fs -ls /");
+
+ Cluster.Instance pigInstance = findPigInstance();
+ Predicate<NodeMetadata> pigClientRole = and(alwaysTrue(),
withIds(pigInstance.getId()));
+
+ Map<? extends NodeMetadata, ExecResponse> responses =
controller.runScriptOnNodesMatching(clusterSpec, pigClientRole, binPig);
+
+ LOG.info("Responses for Statement: " + binPig);
+ for (Map.Entry<? extends NodeMetadata, ExecResponse> entry :
responses.entrySet()) {
+ LOG.info("Node[" + entry.getKey().getId() + "]: " + entry.getValue());
+ }
+
+ assertResponsesContain(responses, binPig, "/hadoop");
+ }
+
+ public static void assertResponsesContain(Map<? extends NodeMetadata,
ExecResponse> responses, Statement statement, String text) {
+ for (Map.Entry<? extends NodeMetadata, ExecResponse> entry :
responses.entrySet()) {
+ if (!entry.getValue().getOutput().contains(text)) {
+ failNotEquals("Node: " + entry.getKey().getId()
+ + " failed to execute the command: " + statement
+ + " as could not find expected text", text, entry.getValue());
+ }
+ }
+ }
+
+ private Cluster.Instance findPigInstance() throws IOException {
+ Cluster cluster = new
ClusterStateStoreFactory().create(clusterSpec).load();
+ return cluster.getInstanceMatching(anyRoleIn(newHashSet(PIG_CLIENT_ROLE)));
+ }
+}
Added: whirr/trunk/services/pig/src/test/resources/log4j.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/pig/src/test/resources/log4j.xml?rev=1298301&view=auto
==============================================================================
--- whirr/trunk/services/pig/src/test/resources/log4j.xml (added)
+++ whirr/trunk/services/pig/src/test/resources/log4j.xml Thu Mar 8 08:50:59
2012
@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+ <!--
+ For more configuration infromation and examples see the Apache
+ Log4j website: http://logging.apache.org/log4j/
+ -->
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";
+ debug="false">
+
+ <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+ <param name="Threshold" value="INFO" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WIREFILE"
class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-wire.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="BLOBSTOREFILE"
class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-blobstore.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+
+ <!-- A time/date based rolling appender -->
+ <appender name="COMPUTEFILE"
class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-compute.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WHIRRFILE"
class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/whirr.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="SSHFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-ssh.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+ <appender name="ASYNCCOMPUTE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="COMPUTEFILE" />
+ </appender>
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="FILE" />
+ </appender>
+
+ <appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="WIREFILE" />
+ </appender>
+
+ <appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="BLOBSTOREFILE" />
+ </appender>
+
+ <appender name="ASYNCSSH" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="SSHFILE" />
+ </appender>
+ <!-- ================ -->
+ <!-- Limit categories -->
+ <!-- ================ -->
+ <category name="jclouds.blobstore">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCBLOBSTORE" />
+ </category>
+
+ <category name="org.jclouds">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNC" />
+ </category>
+
+ <category name="org.apache.whirr">
+ <priority value="DEBUG" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
+ <category name="jclouds.headers">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+ <category name="jclouds.compute">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCCOMPUTE" />
+ </category>
+
+ <category name="jclouds.ssh">
+ <priority value="TRACE" />
+ <appender-ref ref="ASYNCSSH" />
+ </category>
+
+ <category name="jclouds.wire">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category><!--
+
+ <category name="jclouds.signature">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+
+
+ --><!-- ======================= -->
+ <!-- Setup the Root category -->
+ <!-- ======================= -->
+
+ <root>
+ <priority value="WARN" />
+ <appender-ref ref="CONSOLE" />
+ </root>
+
+</log4j:configuration>
Added: whirr/trunk/services/pig/src/test/resources/whirr-pig-test.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/pig/src/test/resources/whirr-pig-test.properties?rev=1298301&view=auto
==============================================================================
--- whirr/trunk/services/pig/src/test/resources/whirr-pig-test.properties
(added)
+++ whirr/trunk/services/pig/src/test/resources/whirr-pig-test.properties Thu
Mar 8 08:50:59 2012
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+whirr.cluster-name=pig-itest-${sys:whirr.test.provider}-${sys:user.name}
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+pig-client,1
hadoop-datanode+hadoop-tasktracker
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.pig.version=0.9.2
+whirr.pig.tarball.url=http://apache.osuosl.org/pig/pig-${whirr.pig.version}/pig-${whirr.pig.version}.tar.gz
+
+whirr.hadoop.version=0.20.205.0
+whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
