Author: asavu
Date: Tue Oct 11 10:36:36 2011
New Revision: 1181696
URL: http://svn.apache.org/viewvc?rev=1181696&view=rev
Log:
WHIRR-368. Add the ability to adjust contents of hadoop-env.sh from a cluster
properties file. Contributed by Karel Vervaeke
Modified:
whirr/trunk/CHANGES.txt
whirr/trunk/recipes/hbase-ec2-0.90.properties
whirr/trunk/recipes/hbase-ec2-cdh.properties
whirr/trunk/recipes/hbase-ec2.properties
whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hbase.sh
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh
whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseConfigurationBuilder.java
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java
whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh
whirr/trunk/services/hbase/src/main/resources/whirr-hbase-default.properties
Modified: whirr/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/whirr/trunk/CHANGES.txt?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
--- whirr/trunk/CHANGES.txt (original)
+++ whirr/trunk/CHANGES.txt Tue Oct 11 10:36:36 2011
@@ -29,6 +29,9 @@ Trunk (unreleased changes)
WHIRR-395. Less verbose logging when setting firewall rules.
(asavu via tomwhite)
+ WHIRR-368. Add the ability to adjust contents of hadoop-env.sh from
+ a cluster properties file (Karel Vervaeke via asavu)
+
BUG FIXES
WHIRR-377. Fix broken CLI logging config. (asavu via tomwhite)
Modified: whirr/trunk/recipes/hbase-ec2-0.90.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/recipes/hbase-ec2-0.90.properties?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
--- whirr/trunk/recipes/hbase-ec2-0.90.properties (original)
+++ whirr/trunk/recipes/hbase-ec2-0.90.properties Tue Oct 11 10:36:36 2011
@@ -56,3 +56,7 @@ whirr.hbase.tarball.url=http://apache.cu
# This example uses Cloudera's CDH3.
whirr.hadoop.tarball.url=http://archive.cloudera.com/cdh/3/hadoop-0.20.2-cdh3u1.tar.gz
+# Options for the hbase master & regionserver processes
+#hbase-env.HBASE_MASTER_OPTS=-Xms1000m -Xmx1000m -Xmn256m
-XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails
-XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-master-gc.log
+#hbase-env.HBASE_MASTER_OPTS=-Xms2000m -Xmx2000m -Xmn256m
-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88
-XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/data/hbase/logs/hbase-regionserver-gc.log
+
Modified: whirr/trunk/recipes/hbase-ec2-cdh.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/recipes/hbase-ec2-cdh.properties?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
--- whirr/trunk/recipes/hbase-ec2-cdh.properties (original)
+++ whirr/trunk/recipes/hbase-ec2-cdh.properties Tue Oct 11 10:36:36 2011
@@ -55,3 +55,7 @@ whirr.location-id=us-east-1
# By default use the user system SSH keys. Override them here.
# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
# whirr.public-key-file=${whirr.private-key-file}.pub
+
+# Options for the hbase master & regionserver processes
+#hbase-env.HBASE_MASTER_OPTS=-Xms1000m -Xmx1000m -Xmn256m
-XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails
-XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-master-gc.log
+#hbase-env.HBASE_REGIONSERVER_OPTS=-Xms2000m -Xmx2000m -Xmn256m
-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88
-XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/data/hbase/logs/hbase-regionserver-gc.log
Modified: whirr/trunk/recipes/hbase-ec2.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/recipes/hbase-ec2.properties?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
--- whirr/trunk/recipes/hbase-ec2.properties (original)
+++ whirr/trunk/recipes/hbase-ec2.properties Tue Oct 11 10:36:36 2011
@@ -46,3 +46,8 @@ whirr.location-id=us-east-1
# Expert: specify the version of HBase to install.
#whirr.hbase.tarball.url=http://archive.apache.org/dist/hbase/hbase-0.89.20100924/hbase-0.89.20100924-bin.tar.gz
+
+# Options for the hbase master & regionserver processes
+#hbase-env.HBASE_MASTER_OPTS=-Xms1000m -Xmx1000m -Xmn256m
-XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails
-XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-master-gc.log
+#hbase-env.HBASE_REGIONSERVER_OPTS=-Xms2000m -Xmx2000m -Xmn256m
-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88
-XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/data/hbase/logs/hbase-regionserver-gc.log
+
Modified:
whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
(original)
+++
whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
Tue Oct 11 10:36:36 2011
@@ -51,6 +51,7 @@ function configure_cdh_hadoop() {
# Copy generated configuration files in place
cp /tmp/{core,hdfs,mapred}-site.xml $HADOOP_CONF_DIR
+ cp /tmp/hadoop-env.sh $HADOOP_CONF_DIR
# Expose /metrics URL endpoint
cat > $HADOOP_CONF_DIR/hadoop-metrics.properties <<EOF
@@ -61,21 +62,26 @@ jvm.class=org.apache.hadoop.metrics.spi.
rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
EOF
- # Set SSH options within the cluster
- sed -i -e 's|# export HADOOP_SSH_OPTS=.*|export HADOOP_SSH_OPTS="-o
StrictHostKeyChecking=no"|' \
- $HADOOP_CONF_DIR/hadoop-env.sh
-
- # Disable IPv6
- sed -i -e 's|# Extra Java runtime options. Empty by default.|# Extra Java
runtime options. Empty by default.\nexport HADOOP_OPTS="$HADOOP_OPTS
-Djava.net.preferIPv4Stack=true"|' \
- $HADOOP_CONF_DIR/hadoop-env.sh
+ # Keep PID files in a non-temporary directory
+ HADOOP_PID_DIR=$(. /tmp/hadoop-env.sh; echo $HADOOP_PID_DIR)
+ HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/run/hadoop}
+ mkdir -p $HADOOP_PID_DIR
+ chgrp -R hadoop $HADOOP_PID_DIR
+ chmod -R g+w $HADOOP_PID_DIR
- # Hadoop logs should be on the /data partition
- rm -rf /var/log/hadoop-0.20
+ # Create the actual log dir
mkdir -p /data/hadoop/logs
- chmod g+w /data/hadoop/logs
chgrp -R hadoop /data/hadoop/logs
- ln -s /data/hadoop/logs /var/log/hadoop-0.20
- chgrp -R hadoop /var/log/hadoop /var/log/hadoop-0.20
+ chmod -R g+w /data/hadoop/logs
+
+ # Create a symlink at $HADOOP_LOG_DIR
+ HADOOP_LOG_DIR=$(. /tmp/hadoop-env.sh; echo $HADOOP_LOG_DIR)
+ HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-/var/log/hadoop/logs}
+ rm -rf $HADOOP_LOG_DIR
+ mkdir -p $(dirname $HADOOP_LOG_DIR)
+ ln -s /data/hadoop/logs $HADOOP_LOG_DIR
+ chgrp -R hadoop $HADOOP_LOG_DIR
+ chmod -R $HADOOP_LOG_DIR
for role in $(echo "$ROLES" | tr "," "\n"); do
case $role in
Modified:
whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hbase.sh
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hbase.sh?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hbase.sh
(original)
+++
whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hbase.sh
Tue Oct 11 10:36:36 2011
@@ -77,14 +77,27 @@ function configure_cdh_hbase() {
chmod a+rwxt /data/tmp
fi
- # Copy generated configuration file in place
+ # Copy generated configuration files in place
cp /tmp/hbase-site.xml $HBASE_CONF_DIR
+ cp /tmp/hbase-env.sh $HBASE_CONF_DIR
- # override JVM options
- cat >> $HBASE_CONF_DIR/hbase-env.sh <<EOF
-export HBASE_MASTER_OPTS="-Xms1000m -Xmx1000m -Xmn256m -XX:+UseConcMarkSweepGC
-XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/data/hbase/logs/hbase-master-gc.log"
-export HBASE_REGIONSERVER_OPTS="-Xms2000m -Xmx2000m -Xmn256m
-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88
-XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/data/hbase/logs/hbase-regionserver-gc.log"
-EOF
+ # HBASE_PID_DIR should exist and be owned by hadoop:hadoop
+ HBASE_PID_DIR=$(. $HBASE_CONF_DIR/hbase-env.sh; echo $HBASE_PID_DIR)
+ HBASE_PID_DIR=${HBASE_PID_DIR:-/var/run/hbase}
+ mkdir -p $HBASE_PID_DIR
+ chown -R hadoop:hadoop $HBASE_PID_DIR
+
+ # Create the actual log dir
+ mkdir -p /data/hbase/logs
+ chown -R hbase:hbase /data/hbase/logs
+
+ # Create a symlink at $HBASE_LOG_DIR
+ HBASE_LOG_DIR=$(. $HBASE_CONF_DIR/hbase-env.sh; echo $HBASE_LOG_DIR)
+ HBASE_LOG_DIR=${HBASE_LOG_DIR:-/var/log/hbase}
+ rm -rf $HBASE_LOG_DIR
+ mkdir -p $(dirname $HBASE_LOG_DIR)
+ ln -s /data/hbase/logs $HBASE_LOG_DIR
+ chown -R hbase:hbase $HBASE_LOG_DIR
# configure hbase for ganglia
cat > $HBASE_CONF_DIR/hadoop-metrics.properties <<EOF
@@ -99,27 +112,6 @@ jvm.period=10
jvm.servers=$MASTER_HOST:8649
EOF
- # keep PID files in a non-temporary directory
- sed -i -e "s|# export HBASE_PID_DIR=.*|export HBASE_PID_DIR=/var/run/hbase|"
\
- $HBASE_CONF_DIR/hbase-env.sh
-
- # set SSH options within the cluster
- sed -i -e 's|# export HBASE_SSH_OPTS=.*|export HBASE_SSH_OPTS="-o
StrictHostKeyChecking=no"|' \
- $HBASE_CONF_DIR/hbase-env.sh
-
- # disable IPv6
- sed -i -e 's|export HBASE_OPTS="$HBASE_OPTS -ea -XX:+UseConcMarkSweepGC
-XX:+CMSIncrementalMode"|export HBASE_OPTS="$HBASE_OPTS -ea
-XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode
-Djava.net.preferIPv4Stack=true"|' \
- $HBASE_CONF_DIR/hbase-env.sh
-
- # hbase logs should be on the /data partition
- sed -i -e 's|# export HBASE_LOG_DIR=.*|export
HBASE_LOG_DIR=/var/log/hbase/logs|' \
- $HBASE_CONF_DIR/hbase-env.sh
- rm -rf /var/log/hbase
- mkdir /data/hbase/logs
- chown hbase:hbase /data/hbase/logs
- ln -s /data/hbase/logs /var/log/hbase
- chown -R hbase:hbase /var/log/hbase
-
# Now that the configuration is done, install the daemon packages
for role in $(echo "$ROLES" | tr "," "\n"); do
case $role in
Modified:
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java
(original)
+++
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java
Tue Oct 11 10:36:36 2011
@@ -56,21 +56,28 @@ public class HadoopConfigurationBuilder
Cluster cluster) throws ConfigurationException, IOException {
Configuration config = buildCommonConfiguration(clusterSpec, cluster,
new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES));
- return HadoopConfigurationConverter.asCreateFileStatement(path, config);
+ return
HadoopConfigurationConverter.asCreateXmlConfigurationFileStatement(path,
config);
}
public static Statement buildHdfs(String path, ClusterSpec clusterSpec,
Cluster cluster) throws ConfigurationException, IOException {
Configuration config = buildHdfsConfiguration(clusterSpec, cluster,
new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES));
- return HadoopConfigurationConverter.asCreateFileStatement(path, config);
+ return
HadoopConfigurationConverter.asCreateXmlConfigurationFileStatement(path,
config);
}
public static Statement buildMapReduce(String path, ClusterSpec clusterSpec,
Cluster cluster) throws ConfigurationException, IOException {
Configuration config = buildMapReduceConfiguration(clusterSpec, cluster,
new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES));
- return HadoopConfigurationConverter.asCreateFileStatement(path, config);
+ return
HadoopConfigurationConverter.asCreateXmlConfigurationFileStatement(path,
config);
+ }
+
+ public static Statement buildHadoopEnv(String path, ClusterSpec clusterSpec,
+ Cluster cluster) throws ConfigurationException, IOException {
+ Configuration config = buildHadoopEnvConfiguration(clusterSpec, cluster,
+ new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES));
+ return
HadoopConfigurationConverter.asCreateEnvironmentVariablesFileStatement(path,
config);
}
@VisibleForTesting
@@ -131,6 +138,12 @@ public class HadoopConfigurationBuilder
jobtracker.getPublicAddress().getHostName()));
return config;
}
+
+ @VisibleForTesting
+ static Configuration buildHadoopEnvConfiguration(ClusterSpec clusterSpec,
+ Cluster cluster, Configuration defaults) throws ConfigurationException {
+ return build(clusterSpec, cluster, defaults, "hadoop-env");
+ }
private static void setIfAbsent(Configuration config, String property,
String value) {
if (!config.containsKey(property)) {
Modified:
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java
(original)
+++
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java
Tue Oct 11 10:36:36 2011
@@ -38,7 +38,7 @@ public class HadoopConfigurationConverte
private static final String FINAL_SUFFIX = ".final";
@VisibleForTesting
- static List<String> asLinesInFile(Configuration hadoopConfig) {
+ static List<String> asXmlConfigurationLines(Configuration hadoopConfig) {
List<String> lines = Lists.newArrayList();
lines.add("<configuration>");
for (@SuppressWarnings("unchecked")
@@ -64,9 +64,36 @@ public class HadoopConfigurationConverte
return lines;
}
- public static Statement asCreateFileStatement(String path,
+ public static Statement asCreateXmlConfigurationFileStatement(String path,
Configuration hadoopConfig) {
- return Statements.appendFile(path, asLinesInFile(hadoopConfig));
+ return Statements.appendFile(path, asXmlConfigurationLines(hadoopConfig));
}
+ @VisibleForTesting
+ static List<String> asEnvironmentVariablesLines(Configuration hadoopConfig) {
+ List<String> lines = Lists.newArrayList();
+
+ for (@SuppressWarnings("unchecked")
+ Iterator<String> it = hadoopConfig.getKeys(); it.hasNext(); ) {
+ String key = it.next();
+ if (key.endsWith(FINAL_SUFFIX)) {
+ continue;
+ }
+
+ // Write the export line. We only allow one value per key
+ String value = hadoopConfig.getString(key);
+ lines.add(new StringBuilder("export ")
+ .append(key)
+ .append("=\"")
+ .append(value)
+ .append("\"").toString());
+ }
+ return lines;
+ }
+
+ public static Statement asCreateEnvironmentVariablesFileStatement(String
path,
+ Configuration config) {
+ return Statements.appendFile(path, asEnvironmentVariablesLines(config));
+ }
+
}
Modified:
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java
(original)
+++
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java
Tue Oct 11 10:36:36 2011
@@ -21,6 +21,7 @@ package org.apache.whirr.service.hadoop;
import static
org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildCommon;
import static
org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHdfs;
import static
org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildMapReduce;
+import static
org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHadoopEnv;
import static org.jclouds.scriptbuilder.domain.Statements.call;
import java.io.IOException;
@@ -49,7 +50,8 @@ public class HadoopDataNodeClusterAction
event.getStatementBuilder().addStatements(
buildCommon("/tmp/core-site.xml", clusterSpec, cluster),
buildHdfs("/tmp/hdfs-site.xml", clusterSpec, cluster),
- buildMapReduce("/tmp/mapred-site.xml", clusterSpec, cluster)
+ buildMapReduce("/tmp/mapred-site.xml", clusterSpec, cluster),
+ buildHadoopEnv("/tmp/hadoop-env.sh", clusterSpec, cluster)
);
} catch (ConfigurationException e) {
throw new IOException(e);
Modified:
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
(original)
+++
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
Tue Oct 11 10:36:36 2011
@@ -20,6 +20,7 @@ package org.apache.whirr.service.hadoop;
import static org.apache.whirr.RolePredicates.role;
import static
org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildCommon;
+import static
org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHadoopEnv;
import static
org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHdfs;
import static
org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildMapReduce;
import static org.jclouds.scriptbuilder.domain.Statements.call;
@@ -80,7 +81,8 @@ public class HadoopNameNodeClusterAction
event.getStatementBuilder().addStatements(
buildCommon("/tmp/core-site.xml", clusterSpec, cluster),
buildHdfs("/tmp/hdfs-site.xml", clusterSpec, cluster),
- buildMapReduce("/tmp/mapred-site.xml", clusterSpec, cluster)
+ buildMapReduce("/tmp/mapred-site.xml", clusterSpec, cluster),
+ buildHadoopEnv("/tmp/hadoop-env.sh", clusterSpec, cluster)
);
} catch (ConfigurationException e) {
throw new IOException(e);
Modified:
whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh
(original)
+++
whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh
Tue Oct 11 10:36:36 2011
@@ -53,30 +53,26 @@ function configure_hadoop() {
# Copy generated configuration files in place
cp /tmp/{core,hdfs,mapred}-site.xml $HADOOP_CONF_DIR
+ cp /tmp/hadoop-env.sh $HADOOP_CONF_DIR
# Keep PID files in a non-temporary directory
- sed -i -e "s|# export HADOOP_PID_DIR=.*|export
HADOOP_PID_DIR=/var/run/hadoop|" \
- $HADOOP_CONF_DIR/hadoop-env.sh
- mkdir -p /var/run/hadoop
- chown -R hadoop:hadoop /var/run/hadoop
-
- # Set SSH options within the cluster
- sed -i -e 's|# export HADOOP_SSH_OPTS=.*|export HADOOP_SSH_OPTS="-o
StrictHostKeyChecking=no"|' \
- $HADOOP_CONF_DIR/hadoop-env.sh
-
- # Disable IPv6
- sed -i -e 's|# export HADOOP_OPTS=.*|export
HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"|' \
- $HADOOP_CONF_DIR/hadoop-env.sh
-
- # Hadoop logs should be on the /data partition
- sed -i -e 's|# export HADOOP_LOG_DIR=.*|export
HADOOP_LOG_DIR=/var/log/hadoop/logs|' \
- $HADOOP_CONF_DIR/hadoop-env.sh
- rm -rf /var/log/hadoop
- mkdir /data/hadoop/logs
- chown hadoop:hadoop /data/hadoop/logs
- ln -s /data/hadoop/logs /var/log/hadoop
- chown -R hadoop:hadoop /var/log/hadoop
-
+ HADOOP_PID_DIR=$(. /tmp/hadoop-env.sh; echo $HADOOP_PID_DIR)
+ HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/run/hadoop}
+ mkdir -p $HADOOP_PID_DIR
+ chown -R hadoop:hadoop $HADOOP_PID_DIR
+
+ # Create the actual log dir
+ mkdir -p /data/hadoop/logs
+ chown -R hadoop:hadoop /data/hadoop/logs
+
+ # Create a symlink at $HADOOP_LOG_DIR
+ HADOOP_LOG_DIR=$(. /tmp/hadoop-env.sh; echo $HADOOP_LOG_DIR)
+ HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-/var/log/hadoop/logs}
+ rm -rf $HADOOP_LOG_DIR
+ mkdir -p $(dirname $HADOOP_LOG_DIR)
+ ln -s /data/hadoop/logs $HADOOP_LOG_DIR
+ chown -R hadoop:hadoop $HADOOP_LOG_DIR
+
for role in $(echo "$ROLES" | tr "," "\n"); do
case $role in
hadoop-namenode)
Modified:
whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
(original)
+++
whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
Tue Oct 11 10:36:36 2011
@@ -47,3 +47,13 @@ hadoop-mapreduce.mapred.compress.map.out
hadoop-mapreduce.mapred.output.compression.type=BLOCK
hadoop-mapreduce.mapred.child.java.opts=-Xmx550m
hadoop-mapreduce.mapred.child.ulimit=1126400
+
+# hadoop-env.sh
+# keep PID files in a non-temporary directory
+hadoop-env.HADOOP_PID_DIR=/var/run/hadoop
+# set SSH options within the cluster
+hadoop-env.HADOOP_SSH_OPTS=-o StrictHostKeyChecking=no
+# disable ipv6
+hadoop-env.HADOOP_OPTS=-Djava.net.preferIPv4Stack=true
+# Log dir. Note: This will be a symlink. The actual location is determined in
the install function.
+hadoop-env.HADOOP_LOG_DIR=/var/log/hadoop/logs
Modified:
whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java
(original)
+++
whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java
Tue Oct 11 10:36:36 2011
@@ -36,7 +36,7 @@ public class HadoopConfigurationConverte
Configuration conf = new PropertiesConfiguration();
conf.setProperty("p1", "v1,v2");
conf.setProperty("p2", "v3");
- List<String> lines = HadoopConfigurationConverter.asLinesInFile(conf);
+ List<String> lines =
HadoopConfigurationConverter.asXmlConfigurationLines(conf);
assertThat(lines, is((List<String>) Lists.newArrayList(
"<configuration>",
" <property>",
@@ -57,7 +57,7 @@ public class HadoopConfigurationConverte
conf.setProperty("p1", "v1");
conf.setProperty("p2", "v2");
conf.setProperty("p2.final", "true");
- List<String> lines = HadoopConfigurationConverter.asLinesInFile(conf);
+ List<String> lines =
HadoopConfigurationConverter.asXmlConfigurationLines(conf);
assertThat(lines, is((List<String>) Lists.newArrayList(
"<configuration>",
" <property>",
Modified:
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseConfigurationBuilder.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseConfigurationBuilder.java?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseConfigurationBuilder.java
(original)
+++
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseConfigurationBuilder.java
Tue Oct 11 10:36:36 2011
@@ -37,7 +37,7 @@ public class HBaseConfigurationBuilder {
throws ConfigurationException, IOException {
Configuration config = buildHBaseSiteConfiguration(clusterSpec, cluster,
new
PropertiesConfiguration(HBaseConstants.FILE_HBASE_DEFAULT_PROPERTIES));
- return HadoopConfigurationConverter.asCreateFileStatement(path, config);
+ return
HadoopConfigurationConverter.asCreateXmlConfigurationFileStatement(path,
config);
}
static Configuration buildHBaseSiteConfiguration(ClusterSpec clusterSpec,
Cluster cluster, Configuration defaults)
@@ -54,6 +54,20 @@ public class HBaseConfigurationBuilder {
return config;
}
+ public static Statement buildHBaseEnv(String path, ClusterSpec clusterSpec,
Cluster cluster)
+ throws ConfigurationException, IOException {
+ Configuration config = buildHBaseEnvConfiguration(clusterSpec, cluster,
+ new
PropertiesConfiguration(HBaseConstants.FILE_HBASE_DEFAULT_PROPERTIES));
+ return
HadoopConfigurationConverter.asCreateEnvironmentVariablesFileStatement(path,
config);
+ }
+
+ static Configuration buildHBaseEnvConfiguration(ClusterSpec clusterSpec,
Cluster cluster, Configuration defaults)
+ throws ConfigurationException, IOException {
+ Configuration config = build(clusterSpec, cluster, defaults, "hbase-env");
+
+ return config;
+ }
+
private static Configuration build(ClusterSpec clusterSpec, Cluster cluster,
Configuration defaults, String prefix)
throws ConfigurationException {
CompositeConfiguration config = new CompositeConfiguration();
Modified:
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java
(original)
+++
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java
Tue Oct 11 10:36:36 2011
@@ -19,7 +19,7 @@
package org.apache.whirr.service.hbase;
import static org.apache.whirr.RolePredicates.role;
-import static org.apache.whirr.service.FirewallManager.Rule;
+import static
org.apache.whirr.service.hbase.HBaseConfigurationBuilder.buildHBaseEnv;
import static
org.apache.whirr.service.hbase.HBaseConfigurationBuilder.buildHBaseSite;
import static org.jclouds.scriptbuilder.domain.Statements.call;
@@ -33,11 +33,13 @@ import java.net.InetAddress;
import java.util.Map.Entry;
import java.util.Properties;
+import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.whirr.Cluster;
import org.apache.whirr.Cluster.Instance;
import org.apache.whirr.ClusterSpec;
import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.FirewallManager.Rule;
import org.apache.whirr.service.hadoop.HadoopProxy;
import org.apache.whirr.service.zookeeper.ZooKeeperCluster;
import org.slf4j.Logger;
@@ -82,6 +84,7 @@ public class HBaseMasterClusterActionHan
protected void beforeConfigure(ClusterActionEvent event) throws IOException,
InterruptedException {
ClusterSpec clusterSpec = event.getClusterSpec();
Cluster cluster = event.getCluster();
+ Configuration conf = getConfiguration(clusterSpec);
LOG.info("Authorizing firewall");
Instance instance = cluster.getInstanceMatching(role(ROLE));
@@ -94,8 +97,9 @@ public class HBaseMasterClusterActionHan
);
try {
- event.getStatementBuilder().addStatement(
- buildHBaseSite("/tmp/hbase-site.xml", clusterSpec, cluster)
+ event.getStatementBuilder().addStatements(
+ buildHBaseSite("/tmp/hbase-site.xml", clusterSpec, cluster),
+ buildHBaseEnv("/tmp/hbase-env.sh", clusterSpec, cluster)
);
} catch (ConfigurationException e) {
throw new IOException(e);
@@ -105,16 +109,16 @@ public class HBaseMasterClusterActionHan
String quorum = ZooKeeperCluster.getHosts(cluster);
String tarurl = prepareRemoteFileUrl(event,
- getConfiguration(clusterSpec).getString(HBaseConstants.KEY_TARBALL_URL));
+ conf.getString(HBaseConstants.KEY_TARBALL_URL));
addStatement(event, call(
- getConfigureFunction(getConfiguration(clusterSpec)),
+ getConfigureFunction(conf),
ROLE,
HBaseConstants.PARAM_MASTER, master,
HBaseConstants.PARAM_QUORUM, quorum,
HBaseConstants.PARAM_PROVIDER, clusterSpec.getProvider(),
- HBaseConstants.PARAM_TARBALL_URL, tarurl)
- );
+ HBaseConstants.PARAM_TARBALL_URL, tarurl
+ ));
}
@Override
Modified:
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java
(original)
+++
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java
Tue Oct 11 10:36:36 2011
@@ -20,12 +20,14 @@ package org.apache.whirr.service.hbase;
import static org.apache.whirr.RolePredicates.role;
import static org.apache.whirr.service.FirewallManager.Rule;
+import static
org.apache.whirr.service.hbase.HBaseConfigurationBuilder.buildHBaseEnv;
import static
org.apache.whirr.service.hbase.HBaseConfigurationBuilder.buildHBaseSite;
import static org.jclouds.scriptbuilder.domain.Statements.call;
import java.io.IOException;
import java.net.InetAddress;
+import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.whirr.Cluster;
import org.apache.whirr.Cluster.Instance;
@@ -70,6 +72,7 @@ public class HBaseRegionServerClusterAct
throws IOException, InterruptedException {
ClusterSpec clusterSpec = event.getClusterSpec();
Cluster cluster = event.getCluster();
+ Configuration conf = getConfiguration(clusterSpec);
Instance instance = cluster.getInstanceMatching(
role(HBaseMasterClusterActionHandler.ROLE));
@@ -82,8 +85,9 @@ public class HBaseRegionServerClusterAct
);
try {
- event.getStatementBuilder().addStatement(
- buildHBaseSite("/tmp/hbase-site.xml", clusterSpec, cluster)
+ event.getStatementBuilder().addStatements(
+ buildHBaseSite("/tmp/hbase-site.xml", clusterSpec, cluster),
+ buildHBaseEnv("/tmp/hbase-env.sh", clusterSpec, cluster)
);
} catch (ConfigurationException e) {
throw new IOException(e);
@@ -93,10 +97,10 @@ public class HBaseRegionServerClusterAct
String quorum = ZooKeeperCluster.getHosts(cluster);
String tarurl = prepareRemoteFileUrl(event,
- getConfiguration(clusterSpec).getString(HBaseConstants.KEY_TARBALL_URL));
+ conf.getString(HBaseConstants.KEY_TARBALL_URL));
addStatement(event, call(
- getConfigureFunction(getConfiguration(clusterSpec)),
+ getConfigureFunction(conf),
ROLE,
HBaseConstants.PARAM_MASTER, master,
HBaseConstants.PARAM_QUORUM, quorum,
Modified:
whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
--- whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh
(original)
+++ whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh
Tue Oct 11 10:36:36 2011
@@ -91,14 +91,25 @@ function configure_hbase() {
mkdir /etc/hbase
ln -s $HBASE_CONF_DIR /etc/hbase/conf
- # Copy generated configuration file in place
+ # Copy generated configuration files in place
cp /tmp/hbase-site.xml $HBASE_CONF_DIR
+ cp /tmp/hbase-env.sh $HBASE_CONF_DIR
- # override JVM options
- cat >> $HBASE_CONF_DIR/hbase-env.sh <<EOF
-export HBASE_MASTER_OPTS="-Xms1000m -Xmx1000m -Xmn256m -XX:+UseConcMarkSweepGC
-XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
-export HBASE_REGIONSERVER_OPTS="-Xms2000m -Xmx2000m -Xmn256m
-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88
-XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
-EOF
+ # HBASE_PID_DIR should exist and be owned by hadoop:hadoop
+ mkdir -p /var/run/hbase
+ chown -R hadoop:hadoop /var/run/hbase
+
+ # Create the actual log dir
+ mkdir -p $MOUNT/hbase/logs
+ chown -R hadoop:hadoop $MOUNT/hbase/logs
+
+ # Create a symlink at $HBASE_LOG_DIR
+ HBASE_LOG_DIR=$(. $HBASE_CONF_DIR/hbase-env.sh; echo $HBASE_LOG_DIR)
+ HBASE_LOG_DIR=${HBASE_LOG_DIR:-/var/log/hbase/logs}
+ rm -rf $HBASE_LOG_DIR
+ mkdir -p $(dirname $HBASE_LOG_DIR)
+ ln -s $MOUNT/hbase/logs $HBASE_LOG_DIR
+ chown -R hadoop:hadoop $HBASE_LOG_DIR
# configure hbase for ganglia
cat > $HBASE_CONF_DIR/hadoop-metrics.properties <<EOF
@@ -146,29 +157,6 @@ EOF
echo Copy hadoop jar to HBase error: did not find your Hadoop installation
fi
- # keep PID files in a non-temporary directory
- sed -i -e "s|# export HBASE_PID_DIR=.*|export HBASE_PID_DIR=/var/run/hbase|"
\
- $HBASE_CONF_DIR/hbase-env.sh
- mkdir -p /var/run/hbase
- chown -R hadoop:hadoop /var/run/hbase
-
- # set SSH options within the cluster
- sed -i -e 's|# export HBASE_SSH_OPTS=.*|export HBASE_SSH_OPTS="-o
StrictHostKeyChecking=no"|' \
- $HBASE_CONF_DIR/hbase-env.sh
-
- # disable IPv6
- sed -i -e 's|# export HBASE_OPTS=.*|export
HBASE_OPTS="-Djava.net.preferIPv4Stack=true"|' \
- $HBASE_CONF_DIR/hbase-env.sh
-
- # hbase logs should be on the /mnt partition
- sed -i -e 's|# export HBASE_LOG_DIR=.*|export
HBASE_LOG_DIR=/var/log/hbase/logs|' \
- $HBASE_CONF_DIR/hbase-env.sh
- rm -rf /var/log/hbase
- mkdir $MOUNT/hbase/logs
- chown hadoop:hadoop $MOUNT/hbase/logs
- ln -s $MOUNT/hbase/logs /var/log/hbase
- chown -R hadoop:hadoop /var/log/hbase
-
for role in $(echo "$ROLES" | tr "," "\n"); do
case $role in
hbase-master)
Modified:
whirr/trunk/services/hbase/src/main/resources/whirr-hbase-default.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/resources/whirr-hbase-default.properties?rev=1181696&r1=1181695&r2=1181696&view=diff
==============================================================================
---
whirr/trunk/services/hbase/src/main/resources/whirr-hbase-default.properties
(original)
+++
whirr/trunk/services/hbase/src/main/resources/whirr-hbase-default.properties
Tue Oct 11 10:36:36 2011
@@ -32,3 +32,17 @@ hbase-site.hbase.client.retries.number=1
# hbase.client.retries.number)
hbase-site.hbase.zookeeper.recoverable.waittime=600000
+# hbase-env.sh (Values should not be quoted here, they will be quoted with
double quotes in hbase-env.sh)
+# enable assertions, use CMS in incremental mode, disable ipv6
+hbase-env.HBASE_OPTS=-ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode
-Djava.net.preferIPv4Stack=true
+# keep PID files in a non-temporary directory
+hbase-env.HBASE_PID_DIR=/var/run/hbase
+# set SSH options within the cluster
+hbase-env.HBASE_SSH_OPTS=-o StrictHostKeyChecking=no
+# Log dir. Note: This will be a symlink. The actual location is determined in
the install function.
+hbase-env.HBASE_LOG_DIR=/var/log/hbase/logs
+# master options
+hbase-env.HBASE_MASTER_OPTS=-Xms1000m -Xmx1000m -Xmn256m -verbose:gc
-XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/data/hbase/logs/hbase-master-gc.log
+# regionserver options
+hbase-env.HBASE_REGIONSERVER_OPTS=-Xms2000m -Xmx2000m -Xmn256m
-XX:CMSInitiatingOccupancyFraction=88 -XX:+AggressiveOpts -verbose:gc
-XX:+PrintGCDetails -XX:+PrintGCTimeStamps
-Xloggc:/data/hbase/logs/hbase-regionserver-gc.log
+
