Updated Branches: refs/heads/branch-0.8 c0ad6e869 -> 50ef0ad29
WHIRR-120. Add option to populate /etc/hosts, and translate security groups to iptables rules. Project: http://git-wip-us.apache.org/repos/asf/whirr/repo Commit: http://git-wip-us.apache.org/repos/asf/whirr/commit/50ef0ad2 Tree: http://git-wip-us.apache.org/repos/asf/whirr/tree/50ef0ad2 Diff: http://git-wip-us.apache.org/repos/asf/whirr/diff/50ef0ad2 Branch: refs/heads/branch-0.8 Commit: 50ef0ad29f2ccf10679042c5216a03efe86d881c Parents: c0ad6e8 Author: Andrew Bayer <[email protected]> Authored: Sat Sep 8 16:19:32 2012 -0700 Committer: Andrew Bayer <[email protected]> Committed: Tue Apr 9 14:46:56 2013 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/whirr/ClusterSpec.java | 20 +++- .../whirr/actions/ConfigureServicesAction.java | 4 +- .../whirr/actions/ScriptBasedClusterAction.java | 6 +- .../whirr/service/ClusterActionHandlerSupport.java | 110 +++++++++++---- .../org/apache/whirr/service/FirewallManager.java | 106 ++++++++++++-- .../jclouds/integration/FirewallManagerTest.java | 1 + .../cassandra/CassandraClusterActionHandler.java | 2 + .../service/chef/ChefClusterActionHandler.java | 2 + .../whirr/service/chef/ChefServiceDryRunTest.java | 2 +- .../elasticsearch/ElasticSearchHandler.java | 2 + .../ganglia/GangliaMetadClusterActionHandler.java | 2 + .../GangliaMonitorClusterActionHandler.java | 2 + .../service/hadoop/HadoopClusterActionHandler.java | 2 + .../hama/HamaGroomServerClusterActionHandler.java | 2 + .../hama/HamaMasterClusterActionHandler.java | 2 + .../hbase/BasicServerClusterActionHandler.java | 2 + .../hbase/HBaseMasterClusterActionHandler.java | 3 + .../HBaseRegionServerClusterActionHandler.java | 3 + .../mahout/MahoutClientClusterActionHandler.java | 5 + .../service/pig/PigClientClusterActionHandler.java | 5 + .../service/puppet/PuppetClusterActionHandler.java | 4 +- .../service/solr/SolrClusterActionHandler.java | 2 + .../voldemort/VoldemortClusterActionHandler.java | 2 + .../yarn/MapReduceJobHistoryServerHandler.java | 1 + .../whirr/service/yarn/YarnNodeManagerHandler.java | 4 +- .../service/yarn/YarnResourceManagerHandler.java | 2 + .../zookeeper/ZooKeeperClusterActionHandler.java | 2 + src/site/xdoc/configuration-guide.xml | 12 ++- 28 files changed, 257 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/core/src/main/java/org/apache/whirr/ClusterSpec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/whirr/ClusterSpec.java b/core/src/main/java/org/apache/whirr/ClusterSpec.java index e02c675..198b5e6 100644 --- a/core/src/main/java/org/apache/whirr/ClusterSpec.java +++ b/core/src/main/java/org/apache/whirr/ClusterSpec.java @@ -169,7 +169,7 @@ public class ClusterSpec { FIREWALL_RULES_ROLE(String.class, true, "A comma-separated list of port" + " numbers. E.g. 8080,8181. Replace 'role' with an actual role name"), - + VERSION(String.class, false, ""), RUN_URL_BASE(String.class, false, "The base URL for forming run " + @@ -178,6 +178,9 @@ public class ClusterSpec { TERMINATE_ALL_ON_LAUNCH_FAILURE(Boolean.class, false, "Whether or not to " + "automatically terminate all nodes when cluster launch fails for some reason."), + STORE_CLUSTER_IN_ETC_HOSTS(Boolean.class, false, "Whether or not to " + + "store all cluster IPs and hostnames in /etc/hosts on each node."), + AUTO_HOSTNAME_PREFIX(String.class, false, "If given, used a prefix when automatically " + "generating hostnames. Ignored if AUTO_HOSTNAME_SUFFIX is not also set."), @@ -303,6 +306,7 @@ public class ClusterSpec { private String runUrlBase; private boolean terminateAllOnLaunchFailure; + private boolean storeClusterInEtcHosts; private String awsEc2PlacementGroup; @@ -383,7 +387,10 @@ public class ClusterSpec { setTerminateAllOnLaunchFailure(config.getBoolean( Property.TERMINATE_ALL_ON_LAUNCH_FAILURE.getConfigName(), Boolean.TRUE)); - + + setStoreClusterInEtcHosts(config.getBoolean( + Property.STORE_CLUSTER_IN_ETC_HOSTS.getConfigName(), Boolean.FALSE)); + setAwsEc2PlacementGroup(getString(Property.AWS_EC2_PLACEMENT_GROUP)); setByonNodes(byonNodes); @@ -447,6 +454,7 @@ public class ClusterSpec { r.setRunUrlBase(getRunUrlBase()); r.setTerminateAllOnLaunchFailure(isTerminateAllOnLaunchFailure()); + r.setStoreClusterInEtcHosts(isStoreClusterInEtcHosts()); r.setAwsEc2PlacementGroup(getAwsEc2PlacementGroup()); @@ -815,6 +823,13 @@ public class ClusterSpec { this.terminateAllOnLaunchFailure = terminateAllOnLaunchFailure; } + public boolean isStoreClusterInEtcHosts() { + return storeClusterInEtcHosts; + } + public void setStoreClusterInEtcHosts(boolean storeClusterInEtcHosts) { + this.storeClusterInEtcHosts = storeClusterInEtcHosts; + } + public String getAwsEc2PlacementGroup() { return awsEc2PlacementGroup; } @@ -1093,6 +1108,7 @@ public class ClusterSpec { .add("stateStoreBlob", getStateStoreBlob()) .add("awsEc2SpotPrice", getAwsEc2SpotPrice()) .add("terminateAllOnLauchFailure",isTerminateAllOnLaunchFailure()) + .add("storeClusterInEtcHosts",isStoreClusterInEtcHosts()) .add("awsEc2PlacementGroup",getAwsEc2PlacementGroup()) .add("autoHostnamePrefix",getAutoHostnamePrefix()) .add("autoHostnameSuffix",getAutoHostnameSuffix()) http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/core/src/main/java/org/apache/whirr/actions/ConfigureServicesAction.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/whirr/actions/ConfigureServicesAction.java b/core/src/main/java/org/apache/whirr/actions/ConfigureServicesAction.java index abd1b5a..b5b8f65 100644 --- a/core/src/main/java/org/apache/whirr/actions/ConfigureServicesAction.java +++ b/core/src/main/java/org/apache/whirr/actions/ConfigureServicesAction.java @@ -21,7 +21,6 @@ package org.apache.whirr.actions; import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import org.apache.whirr.ClusterSpec; @@ -68,9 +67,8 @@ public class ConfigureServicesAction extends ScriptBasedClusterAction { /** * Apply the firewall rules specified via configuration. */ - protected void eventSpecificActions(Entry<InstanceTemplate, ClusterActionEvent> entry) + protected void eventSpecificActions(InstanceTemplate instanceTemplate, ClusterActionEvent event) throws IOException { - ClusterActionEvent event = entry.getValue(); ClusterSpec clusterSpec = event.getClusterSpec(); Map<String, List<String>> firewallRules = clusterSpec.getFirewallRules(); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java b/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java index b2a522f..2c781af 100644 --- a/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java +++ b/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java @@ -117,6 +117,7 @@ public abstract class ScriptBasedClusterAction extends ClusterAction { instanceTemplate, newCluster, statementBuilder, getCompute(), firewallManager, velocityEngine); eventMap.put(instanceTemplate, event); + eventSpecificActions(instanceTemplate, event); for (String role : instanceTemplate.getRoles()) { if (roleIsInTarget(role)) { safeGetActionHandler(role).beforeAction(event); @@ -172,7 +173,7 @@ public abstract class ScriptBasedClusterAction extends ClusterAction { continue; // skip if not in the target } - eventSpecificActions(entry); + Cluster cluster = entry.getValue().getCluster(); StatementBuilder statementBuilder = entry.getValue().getStatementBuilder(); @@ -259,8 +260,7 @@ public abstract class ScriptBasedClusterAction extends ClusterAction { }); } - protected void eventSpecificActions( - Map.Entry<InstanceTemplate, ClusterActionEvent> entry) throws IOException { + protected void eventSpecificActions(InstanceTemplate instanceTemplate, ClusterActionEvent event) throws IOException { } protected void postRunScriptsActions( http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/core/src/main/java/org/apache/whirr/service/ClusterActionHandlerSupport.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/whirr/service/ClusterActionHandlerSupport.java b/core/src/main/java/org/apache/whirr/service/ClusterActionHandlerSupport.java index 0e8c432..4b6ee89 100644 --- a/core/src/main/java/org/apache/whirr/service/ClusterActionHandlerSupport.java +++ b/core/src/main/java/org/apache/whirr/service/ClusterActionHandlerSupport.java @@ -18,6 +18,8 @@ package org.apache.whirr.service; +import static org.jclouds.scriptbuilder.domain.Statements.exec; + import java.io.File; import java.io.IOException; import java.net.URI; @@ -28,6 +30,7 @@ import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.whirr.ClusterSpec; +import org.apache.whirr.Cluster.Instance; import org.apache.whirr.service.jclouds.RunUrlStatement; import org.apache.whirr.util.BlobCache; import org.jclouds.scriptbuilder.domain.Statement; @@ -47,10 +50,11 @@ public abstract class ClusterActionHandlerSupport implements ClusterActionHandle LoggerFactory.getLogger(ClusterActionHandler.class); public void beforeAction(ClusterActionEvent event) - throws IOException, InterruptedException{ + throws IOException, InterruptedException{ if (event.getAction().equals(BOOTSTRAP_ACTION)) { beforeBootstrap(event); } else if (event.getAction().equals(CONFIGURE_ACTION)) { + addClusterToEtcHostsAndFirewall(event); beforeConfigure(event); } else if (event.getAction().equals(START_ACTION)) { beforeStart(event); @@ -66,7 +70,7 @@ public abstract class ClusterActionHandlerSupport implements ClusterActionHandle } public void afterAction(ClusterActionEvent event) - throws IOException, InterruptedException { + throws IOException, InterruptedException { if (event.getAction().equals(BOOTSTRAP_ACTION)) { afterBootstrap(event); } else if (event.getAction().equals(CONFIGURE_ACTION)) { @@ -135,7 +139,7 @@ public abstract class ClusterActionHandlerSupport implements ClusterActionHandle * @return The composite configuration. */ protected Configuration getConfiguration( - ClusterSpec clusterSpec, Configuration defaults) { + ClusterSpec clusterSpec, Configuration defaults) { CompositeConfiguration cc = new CompositeConfiguration(); cc.addConfiguration(clusterSpec.getConfiguration()); cc.addConfiguration(defaults); @@ -143,24 +147,24 @@ public abstract class ClusterActionHandlerSupport implements ClusterActionHandle } protected Configuration getConfiguration(ClusterSpec clusterSpec, - String defaultsPropertiesFile) throws IOException { + String defaultsPropertiesFile) throws IOException { try { return getConfiguration(clusterSpec, - new PropertiesConfiguration(getClass().getClassLoader().getResource(defaultsPropertiesFile))); + new PropertiesConfiguration(getClass().getClassLoader().getResource(defaultsPropertiesFile))); } catch(ConfigurationException e) { throw new IOException("Error loading " + defaultsPropertiesFile, e); } - } + } /** * A convenience method for adding a {@link RunUrlStatement} to a * {@link ClusterActionEvent}. */ public static void addRunUrl(ClusterActionEvent event, String runUrl, - String... args) - throws IOException { + String... args) + throws IOException { Statement statement = new RunUrlStatement( - event.getClusterSpec().getRunUrlBase(), runUrl, args); + event.getClusterSpec().getRunUrlBase(), runUrl, args); addStatement(event, statement); } @@ -168,6 +172,54 @@ public abstract class ClusterActionHandlerSupport implements ClusterActionHandle event.getStatementBuilder().addStatement(statement); } + public static void addClusterToEtcHostsAndFirewall(ClusterActionEvent event) throws IOException { + if (event.getClusterSpec().isStoreClusterInEtcHosts()) { + addStatement(event, exec("echo -e '\\n' >> /etc/hosts")); + + for (Instance instance : event.getCluster().getInstances()) { + + // Remove any existing references to this IP from /etc/hosts + addStatement(event, exec(String.format("sed -i -e '/%s/d' /etc/hosts", + instance.getPublicIp()))); + // Add this IP to /etc/hosts + addStatement(event, exec(String.format("echo -e '\\n%s %s' >> /etc/hosts", + instance.getPublicIp(), + instance.getPublicHostName()))); + + // Allow access to this host on all ports from this public IP + addStatement(event, exec(String.format("iptables -I INPUT 1 -p tcp --source %s -j ACCEPT || true", + instance.getPublicIp()))); + + if (instance.getPrivateIp() != null) { + // Allow access to this host on all ports from this private IP + addStatement(event, exec(String.format("iptables -I INPUT 1 -p tcp --source %s -j ACCEPT || true", + instance.getPrivateIp()))); + } + + } + + addStatement(event, exec("test -f /etc/hostname && echo $PUBLIC_HOST_NAME > /etc/hostname || true")); + addStatement(event, exec("test -f /etc/sysconfig/network && sed -i -e \"s/HOSTNAME=.*/HOSTNAME=$PUBLIC_HOST_NAME/\" /etc/sysconfig/network || true")); + addStatement(event, exec("test -f /etc/init.d/hostname && /etc/init.d/hostname restart || hostname $PUBLIC_HOST_NAME")); + addStatement(event, exec("sleep 2")); + + addStatement(event, exec("iptables-save || true")); + } + } + + /** + * Handles firewall rules for a given event. + */ + public static void handleFirewallRules(ClusterActionEvent event) { + ClusterSpec clusterSpec = event.getClusterSpec(); + + for (Statement statement : event.getFirewallManager().getRulesAsStatements()) { + addStatement(event, statement); + } + + event.getFirewallManager().authorizeAllRules(); + } + /** * Prepare the file url for the remote machine. * @@ -178,7 +230,7 @@ public abstract class ClusterActionHandlerSupport implements ClusterActionHandle * @return an URL visible to the install / configure scripts */ public static String prepareRemoteFileUrl(ClusterActionEvent event, String rawUrl) - throws IOException { + throws IOException { if (rawUrl != null && rawUrl.startsWith("file://")) { try { URI uri = new URI(rawUrl); @@ -248,25 +300,25 @@ public abstract class ClusterActionHandlerSupport implements ClusterActionHandle return config.getString(key, defaultFunction); } /** - * this uses the inefficient {@link com.google.common.base.Objects} implementation as the object count will be - * relatively small and therefore efficiency is not a concern. - */ - @Override - public int hashCode() { - return Objects.hashCode(getRole()); - } - - @Override - public boolean equals(Object that) { - if (that == null) - return false; - return Objects.equal(this.toString(), that.toString()); - } - - @Override - public String toString() { - return Objects.toStringHelper(this).add("role", getRole()).toString(); - } + * this uses the inefficient {@link com.google.common.base.Objects} implementation as the object count will be + * relatively small and therefore efficiency is not a concern. + */ + @Override + public int hashCode() { + return Objects.hashCode(getRole()); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + return Objects.equal(this.toString(), that.toString()); + } + + @Override + public String toString() { + return Objects.toStringHelper(this).add("role", getRole()).toString(); + } } http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/core/src/main/java/org/apache/whirr/service/FirewallManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/whirr/service/FirewallManager.java b/core/src/main/java/org/apache/whirr/service/FirewallManager.java index d9f780f..0599e43 100644 --- a/core/src/main/java/org/apache/whirr/service/FirewallManager.java +++ b/core/src/main/java/org/apache/whirr/service/FirewallManager.java @@ -18,6 +18,8 @@ package org.apache.whirr.service; +import static org.jclouds.scriptbuilder.domain.Statements.exec; + import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; @@ -39,6 +41,7 @@ import org.jclouds.openstack.nova.v2_0.domain.Ingress; import org.jclouds.openstack.nova.v2_0.domain.SecurityGroup; import org.jclouds.openstack.nova.v2_0.extensions.SecurityGroupApi; import org.jclouds.javax.annotation.Nullable; +import org.jclouds.scriptbuilder.domain.Statement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +54,39 @@ import com.google.common.collect.Sets; public class FirewallManager { + public static class StoredRule { + private Rule rule; + private List<String> cidrs; + private Set<Instance> instances; + + public StoredRule(Rule rule, List<String> cidrs, Set<Instance> instances) { + this.rule = rule; + this.cidrs = cidrs; + this.instances = instances; + } + + /** + * Get the Rule object for this stored rule. + */ + public Rule rule() { + return rule; + } + + /** + * Get the CIDRs for this stored rule. + */ + public List<String> cidrs() { + return cidrs; + } + + /** + * Get the set of Instances for this stored rule. + */ + public Set<Instance> instances() { + return instances; + } + } + public static class Rule { public static Rule create() { @@ -121,17 +157,19 @@ public class FirewallManager { } private static final Logger LOG = LoggerFactory - .getLogger(FirewallManager.class); + .getLogger(FirewallManager.class); private ComputeServiceContext computeServiceContext; private ClusterSpec clusterSpec; private Cluster cluster; - + private Set<StoredRule> storedRules; + public FirewallManager(ComputeServiceContext computeServiceContext, - ClusterSpec clusterSpec, Cluster cluster) { + ClusterSpec clusterSpec, Cluster cluster) { this.computeServiceContext = computeServiceContext; this.clusterSpec = clusterSpec; this.cluster = cluster; + this.storedRules = Sets.newHashSet(); } public void addRules(Rule... rules) throws IOException { @@ -173,19 +211,59 @@ public class FirewallManager { cidrs = Lists.newArrayList(rule.source + "/32"); } + storedRules.add(new StoredRule(rule, cidrs, instances)); + } + + /** + * Logs information about the StoredRule we're adding + * @param storedRule the StoredRule we're adding + */ + private void logInstanceRules(StoredRule storedRule) { Iterable<String> instanceIds = - Iterables.transform(instances, new Function<Instance, String>() { - @Override - public String apply(@Nullable Instance instance) { - return instance == null ? "<null>" : instance.getId(); + Iterables.transform(storedRule.instances(), new Function<Instance, String>() { + @Override + public String apply(@Nullable Instance instance) { + return instance == null ? "<null>" : instance.getId(); + } + }); + + + + LOG.info("Authorizing firewall ingress to {} on ports {} for {}", + new Object[] { instanceIds, storedRule.rule().ports, storedRule.cidrs() }); + } + + /** + * Authorizes all rules via jclouds security groups interface. + */ + public void authorizeAllRules() { + for (StoredRule storedRule : storedRules) { + logInstanceRules(storedRule); + authorizeIngress(computeServiceContext, storedRule.instances(), + clusterSpec, storedRule.cidrs(), storedRule.rule().ports); + } + } + + /** + * Returns a list of Statements for executing iptables for the stored rules. + * @return List of iptables Statements. + */ + public List<Statement> getRulesAsStatements() { + List<Statement> ruleStatements = Lists.newArrayList(); + + for (StoredRule storedRule : storedRules) { + logInstanceRules(storedRule); + for (String cidr : storedRule.cidrs()) { + for (int port : storedRule.rule().ports) { + ruleStatements.add(exec(String.format("iptables -I INPUT 1 -p tcp --dport %d --source %s -j ACCEPT || true", + port, cidr))); } - }); + } + } - LOG.info("Authorizing firewall ingress to {} on ports {} for {}", - new Object[] { instanceIds, rule.ports, cidrs }); + ruleStatements.add(exec("iptables-save || true")); - authorizeIngress(computeServiceContext, instances, - clusterSpec, cidrs, rule.ports); + return ruleStatements; } /** @@ -204,7 +282,7 @@ public class FirewallManager { } public static void authorizeIngress(ComputeServiceContext computeServiceContext, - Set<Instance> instances, ClusterSpec clusterSpec, List<String> cidrs, int... ports) { + Set<Instance> instances, ClusterSpec clusterSpec, List<String> cidrs, int... ports) { if (EC2ApiMetadata.CONTEXT_TOKEN.isAssignableFrom(computeServiceContext.getBackendType())) { // This code (or something like it) may be added to jclouds (see @@ -218,7 +296,7 @@ public class FirewallManager { try { ec2Client.getSecurityGroupServices() .authorizeSecurityGroupIngressInRegion(region, groupName, - IpProtocol.TCP, port, port, cidr); + IpProtocol.TCP, port, port, cidr); } catch(IllegalStateException e) { LOG.warn(e.getMessage()); /* ignore, it means that this permission was already granted */ http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/core/src/test/java/org/apache/whirr/service/jclouds/integration/FirewallManagerTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/whirr/service/jclouds/integration/FirewallManagerTest.java b/core/src/test/java/org/apache/whirr/service/jclouds/integration/FirewallManagerTest.java index afa66c2..defe820 100644 --- a/core/src/test/java/org/apache/whirr/service/jclouds/integration/FirewallManagerTest.java +++ b/core/src/test/java/org/apache/whirr/service/jclouds/integration/FirewallManagerTest.java @@ -89,6 +89,7 @@ public class FirewallManagerTest { Rule.create().destination(instances).port(23344) ); + manager.authorizeAllRules(); } finally { ec2Client.getSecurityGroupServices() .deleteSecurityGroupInRegion(region, groupName); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java b/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java index fcbef4d..6b6b6c2 100644 --- a/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java +++ b/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java @@ -90,6 +90,8 @@ public class CassandraClusterActionHandler extends ClusterActionHandlerSupport { .ports(CLIENT_PORT, JMX_PORT) ); + handleFirewallRules(event); + setInitialTokensAsEnvironmentVariables(event, instances); List<Instance> seeds = getSeeds(instances); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/chef/src/main/java/org/apache/whirr/service/chef/ChefClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/chef/src/main/java/org/apache/whirr/service/chef/ChefClusterActionHandler.java b/services/chef/src/main/java/org/apache/whirr/service/chef/ChefClusterActionHandler.java index e1fad17..0419ca6 100644 --- a/services/chef/src/main/java/org/apache/whirr/service/chef/ChefClusterActionHandler.java +++ b/services/chef/src/main/java/org/apache/whirr/service/chef/ChefClusterActionHandler.java @@ -74,6 +74,8 @@ public class ChefClusterActionHandler extends ClusterActionHandlerSupport { @Override protected void beforeConfigure(ClusterActionEvent event) throws IOException, InterruptedException { + handleFirewallRules(event); + // if the role is an exact match to the prefix then there is nothing to // do (chef only installation) if (role.equals("")) { http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/chef/src/test/java/org/apache/whirr/service/chef/ChefServiceDryRunTest.java ---------------------------------------------------------------------- diff --git a/services/chef/src/test/java/org/apache/whirr/service/chef/ChefServiceDryRunTest.java b/services/chef/src/test/java/org/apache/whirr/service/chef/ChefServiceDryRunTest.java index 7db6f55..067d093 100644 --- a/services/chef/src/test/java/org/apache/whirr/service/chef/ChefServiceDryRunTest.java +++ b/services/chef/src/test/java/org/apache/whirr/service/chef/ChefServiceDryRunTest.java @@ -60,7 +60,7 @@ public class ChefServiceDryRunTest extends BaseServiceDryRunTest { ClusterSpec chefOnly = newClusterSpecForProperties(ImmutableMap.of("whirr.instance-templates", "1 chef")); DryRun dryRun = launchWithClusterSpec(chefOnly); assertScriptPredicateOnPhase(dryRun, "bootstrap", bootstrapPredicate()); - assertNoEntryForPhase(dryRun, "configure"); + // We now have iptables calls by default in the configure phase. } /** http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java ---------------------------------------------------------------------- diff --git a/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java b/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java index 29c3cd8..3bdc24f 100644 --- a/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java +++ b/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java @@ -67,6 +67,8 @@ public class ElasticSearchHandler extends ClusterActionHandlerSupport { .port(HTTP_CLIENT_PORT) ); + handleFirewallRules(event); + Configuration config = ElasticSearchConfigurationBuilder.buildConfig(spec, cluster); addStatement(event, call("retry_helpers")); addStatement(event, http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/ganglia/src/main/java/org/apache/whirr/service/ganglia/GangliaMetadClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/ganglia/src/main/java/org/apache/whirr/service/ganglia/GangliaMetadClusterActionHandler.java b/services/ganglia/src/main/java/org/apache/whirr/service/ganglia/GangliaMetadClusterActionHandler.java index ec1a255..f73fd9e 100644 --- a/services/ganglia/src/main/java/org/apache/whirr/service/ganglia/GangliaMetadClusterActionHandler.java +++ b/services/ganglia/src/main/java/org/apache/whirr/service/ganglia/GangliaMetadClusterActionHandler.java @@ -86,6 +86,8 @@ public class GangliaMetadClusterActionHandler extends ClusterActionHandlerSuppor Rule.create().destination(role(GANGLIA_METAD_ROLE)).ports(HTTP_PORT, GANGLIA_META_PORT) ); + handleFirewallRules(event); + Configuration config = getConfiguration(clusterSpec); String configureFunction = getConfigureFunction(config); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/ganglia/src/main/java/org/apache/whirr/service/ganglia/GangliaMonitorClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/ganglia/src/main/java/org/apache/whirr/service/ganglia/GangliaMonitorClusterActionHandler.java b/services/ganglia/src/main/java/org/apache/whirr/service/ganglia/GangliaMonitorClusterActionHandler.java index 4944014..b9894a8 100644 --- a/services/ganglia/src/main/java/org/apache/whirr/service/ganglia/GangliaMonitorClusterActionHandler.java +++ b/services/ganglia/src/main/java/org/apache/whirr/service/ganglia/GangliaMonitorClusterActionHandler.java @@ -85,6 +85,8 @@ public class GangliaMonitorClusterActionHandler extends ClusterActionHandlerSupp Rule.create().destination(role(GANGLIA_MONITOR_ROLE)).port(GANGLIA_MONITOR_PORT) ); + handleFirewallRules(event); + Configuration config = getConfiguration(clusterSpec); String configureFunction = getConfigureFunction(config); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java b/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java index 26c60ca..fe80ccf 100644 --- a/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java +++ b/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java @@ -100,6 +100,8 @@ public abstract class HadoopClusterActionHandler extends ClusterActionHandlerSup doBeforeConfigure(event); + handleFirewallRules(event); + createHadoopConfigFiles(event, clusterSpec, cluster); addStatement(event, call("retry_helpers")); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java b/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java index e4fa96f..3cb32d0 100644 --- a/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java +++ b/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java @@ -53,6 +53,8 @@ public class HamaGroomServerClusterActionHandler extends event.getFirewallManager().addRules( Rule.create().destination(instance).ports(GROOMSERVER_PORT)); + handleFirewallRules(event); + String hamaConfigureFunction = getConfiguration(clusterSpec).getString( HamaConstants.KEY_CONFIGURE_FUNCTION, HamaConstants.FUNCTION_POST_CONFIGURE); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java b/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java index 650663a..610360a 100644 --- a/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java +++ b/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java @@ -67,6 +67,8 @@ public class HamaMasterClusterActionHandler extends HamaClusterActionHandler { Rule.create().destination(instance).ports(MASTER_WEB_UI_PORT), Rule.create().destination(instance).ports(MASTER_PORT)); + handleFirewallRules(event); + String hamaConfigureFunction = getConfiguration(clusterSpec).getString( HamaConstants.KEY_CONFIGURE_FUNCTION, HamaConstants.FUNCTION_POST_CONFIGURE); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/hbase/src/main/java/org/apache/whirr/service/hbase/BasicServerClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/hbase/src/main/java/org/apache/whirr/service/hbase/BasicServerClusterActionHandler.java b/services/hbase/src/main/java/org/apache/whirr/service/hbase/BasicServerClusterActionHandler.java index 44123c3..6852d4f 100644 --- a/services/hbase/src/main/java/org/apache/whirr/service/hbase/BasicServerClusterActionHandler.java +++ b/services/hbase/src/main/java/org/apache/whirr/service/hbase/BasicServerClusterActionHandler.java @@ -89,6 +89,8 @@ public class BasicServerClusterActionHandler extends HBaseClusterActionHandler { Rule.create().destination(instance).port(port) ); + handleFirewallRules(event); + String master = masterPublicAddress.getHostName(); String quorum = ZooKeeperCluster.getHosts(cluster); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java b/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java index d9f7ed0..02211a2 100644 --- a/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java +++ b/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java @@ -98,6 +98,9 @@ public class HBaseMasterClusterActionHandler extends HBaseClusterActionHandler { //Velocity is assuming flat classloaders or TCCL to load templates. //This doesn't work in OSGi unless we set the TCCL to the bundle classloader before invocation ClassLoader oldTccl = Thread.currentThread().getContextClassLoader(); + + handleFirewallRules(event); + try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); event.getStatementBuilder().addStatements( http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java b/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java index a6ea33e..d204725 100644 --- a/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java +++ b/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java @@ -89,6 +89,9 @@ public class HBaseRegionServerClusterActionHandler extends HBaseClusterActionHan //Velocity is assuming flat classloaders or TCCL to load templates. //This doesn't work in OSGi unless we set the TCCL to the bundle classloader before invocation ClassLoader oldTccl = Thread.currentThread().getContextClassLoader(); + + handleFirewallRules(event); + try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); event.getStatementBuilder().addStatements( http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/mahout/src/main/java/org/apache/whirr/service/mahout/MahoutClientClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/mahout/src/main/java/org/apache/whirr/service/mahout/MahoutClientClusterActionHandler.java b/services/mahout/src/main/java/org/apache/whirr/service/mahout/MahoutClientClusterActionHandler.java index 65b2146..a6247b1 100644 --- a/services/mahout/src/main/java/org/apache/whirr/service/mahout/MahoutClientClusterActionHandler.java +++ b/services/mahout/src/main/java/org/apache/whirr/service/mahout/MahoutClientClusterActionHandler.java @@ -56,4 +56,9 @@ public class MahoutClientClusterActionHandler extends ClusterActionHandlerSuppor addStatement(event, call("retry_helpers")); addStatement(event, call(MAHOUT_CLIENT_SCRIPT, URL_FLAG, mahoutTarball)); } + + @Override + protected void beforeConfigure(ClusterActionEvent event) throws IOException, InterruptedException { + handleFirewallRules(event); + } } http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java b/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java index 2b89681..b6f891b 100644 --- a/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java +++ b/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java @@ -55,4 +55,9 @@ public class PigClientClusterActionHandler extends ClusterActionHandlerSupport { addStatement(event, call(PIG_CLIENT_SCRIPT, URL_FLAG, pigTarball)); } + + @Override + protected void beforeConfigure(ClusterActionEvent event) throws IOException, InterruptedException { + handleFirewallRules(event); + } } http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java index a9841a5..6b5170f 100644 --- a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java +++ b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java @@ -98,8 +98,10 @@ public class PuppetClusterActionHandler extends PuppetInstallClusterActionHandle } protected void beforeConfigure(ClusterActionEvent event) throws IOException, InterruptedException { + handleFirewallRules(event); + super.beforeConfigure(event); - + if (isLastPuppetRoleIn(event.getInstanceTemplate().getRoles()).apply(getRole())) { Configuration config = event.getClusterSpec().getConfiguration(); Iterable<String> roles = RolesManagedByPuppet.INSTANCE.apply(event.getInstanceTemplate().getRoles()); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/solr/src/main/java/org/apache/whirr/service/solr/SolrClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/solr/src/main/java/org/apache/whirr/service/solr/SolrClusterActionHandler.java b/services/solr/src/main/java/org/apache/whirr/service/solr/SolrClusterActionHandler.java index 9c6c933..4fa87a4 100644 --- a/services/solr/src/main/java/org/apache/whirr/service/solr/SolrClusterActionHandler.java +++ b/services/solr/src/main/java/org/apache/whirr/service/solr/SolrClusterActionHandler.java @@ -115,6 +115,8 @@ public class SolrClusterActionHandler extends ClusterActionHandlerSupport { // Open up Jetty port event.getFirewallManager().addRule(Rule.create().destination(role(SOLR_ROLE)).port(jettyPort)); + + handleFirewallRules(event); } @Override http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortClusterActionHandler.java b/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortClusterActionHandler.java index 4eb55ef..2d791bd 100644 --- a/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortClusterActionHandler.java +++ b/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortClusterActionHandler.java @@ -99,6 +99,8 @@ public class VoldemortClusterActionHandler extends ClusterActionHandlerSupport { .ports(CLIENT_PORT, ADMIN_PORT, HTTP_PORT) ); + handleFirewallRules(event); + String servers = Joiner.on(' ').join(getPrivateIps(cluster.getInstances())); Configuration config = event.getClusterSpec().getConfiguration(); http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java ---------------------------------------------------------------------- diff --git a/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java b/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java index 24a8272..6fcb726 100644 --- a/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java +++ b/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java @@ -40,6 +40,7 @@ public class MapReduceJobHistoryServerHandler extends YarnHandler { InterruptedException { ClusterSpec clusterSpec = event.getClusterSpec(); Configuration conf = getConfiguration(clusterSpec); + handleFirewallRules(event); addStatement(event, call(getStartFunction(conf, "mr_jobhistory", "start_mr_jobhistory"), "historyserver")); } http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java ---------------------------------------------------------------------- diff --git a/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java b/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java index 1a4682a..f652d9d 100644 --- a/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java +++ b/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java @@ -61,7 +61,9 @@ public class YarnNodeManagerHandler extends YarnHandler { ); } } - + + handleFirewallRules(event); + try { event.getStatementBuilder().addStatements( build("/tmp/yarn-site.xml", clusterSpec, cluster, ROLE) http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java ---------------------------------------------------------------------- diff --git a/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java b/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java index ffd8a48..dc6108f 100644 --- a/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java +++ b/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java @@ -72,6 +72,8 @@ public class YarnResourceManagerHandler extends YarnHandler { .destination(resourceManager) .ports(RESOURCE_MANAGER_RPC_PORT) ); + + handleFirewallRules(event); try { event.getStatementBuilder().addStatements( http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java b/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java index 367f455..a42ce94 100644 --- a/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java +++ b/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java @@ -84,6 +84,8 @@ public class ZooKeeperClusterActionHandler extends ClusterActionHandlerSupport { Rule.create().destination(role(ZOOKEEPER_ROLE)).port(CLIENT_PORT) ); + handleFirewallRules(event); + // Pass list of all servers in ensemble to configure script. // Position is significant: i-th server has id i. http://git-wip-us.apache.org/repos/asf/whirr/blob/50ef0ad2/src/site/xdoc/configuration-guide.xml ---------------------------------------------------------------------- diff --git a/src/site/xdoc/configuration-guide.xml b/src/site/xdoc/configuration-guide.xml index 035b511..e30a6fd 100644 --- a/src/site/xdoc/configuration-guide.xml +++ b/src/site/xdoc/configuration-guide.xml @@ -559,7 +559,7 @@ xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd <td>The minimum amount of RAM each instance should have</td> </tr> </table> - <subsection name="Firewall Options"></subsection> + <subsection name="Firewall and DNS-Related Options"></subsection> <table border="0"> <tr valign="top"> <th> @@ -608,6 +608,16 @@ xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd <td>none</td> <td>A comma-separated list of port numbers to open on instances with a specific role. Replace {role} the actual role name. E.g. <tt>whirr.firewall-rules.hbase-master=10101</tt>.</td> </tr> + <tr valign="top"> + <td> + <tt>whirr.store-cluster-in-etc-hosts</tt> + </td> + <td> + <tt>--store-cluster-in-etc-hosts</tt> + </td> + <td>false</td> + <td>Whether to store all cluster IPs and hostnames in /etc/hosts on each node.</td> + </tr> </table> <p> <a name="cloud-provider-config"></a>
