Support loginUser and other loginXxx properties for JcloudsLocations, where the loginUser is used to login if distinct from the user which is specified; and the user is created with admin privileges (and root scrambled) assuming it is distinct from the loginUser. userMetadata can also now be specified from the config file. Includes JcloudsLocation significant tidy (but oh so much more to do), and documentation.
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/31b5c079 Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/31b5c079 Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/31b5c079 Branch: refs/heads/0.4.0 Commit: 31b5c079eae1916c419e3dec91945a52824890be Parents: 89d7750 Author: Alex Heneveld <[email protected]> Authored: Sun Sep 23 13:45:59 2012 +0100 Committer: Alex Heneveld <[email protected]> Committed: Tue Oct 2 09:16:29 2012 +0100 ---------------------------------------------------------------------- .../location/basic/SshMachineLocation.java | 8 +- .../location/basic/jclouds/JcloudsLocation.java | 578 +++++++++++++------ .../location/basic/jclouds/JcloudsUtil.java | 15 +- .../AbstractPortableTemplateBuilder.java | 2 +- .../main/java/brooklyn/util/text/Strings.java | 5 + .../jclouds/JcloudsLocationRebindTest.groovy | 33 ++ ...leJcloudsLocationUserLoginAndConfigTest.java | 221 +++++++ .../basic/jclouds/StandaloneJcloudsTest.java | 135 ++++- 8 files changed, 785 insertions(+), 212 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/31b5c079/core/src/main/java/brooklyn/location/basic/SshMachineLocation.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/location/basic/SshMachineLocation.java b/core/src/main/java/brooklyn/location/basic/SshMachineLocation.java index 09cfb7c..22b4c4d 100644 --- a/core/src/main/java/brooklyn/location/basic/SshMachineLocation.java +++ b/core/src/main/java/brooklyn/location/basic/SshMachineLocation.java @@ -90,9 +90,11 @@ public class SshMachineLocation extends AbstractLocation implements MachineLocat /** properties which are passed to ssh */ public static final Collection<String> SSH_PROPS = ImmutableSet.of( "noStdoutLogging", "noStderrLogging", "logPrefix", "out", "err", "password", - "keyFiles", "publicKey", "privateKey", "privateKeyPassphrase", "privateKeyFile", "privateKeyData", - //TODO prefer privateKeyData/privateKeyFile (confusion about whether other holds a file or data) - "permissions", "sshTries", "env", "allocatePTY"); + "permissions", "sshTries", "env", "allocatePTY", + "privateKeyPassphrase", "privateKeyFile", "privateKeyData", + // would like to deprecate these -- prefer privateKeyData/privateKeyFile (confusion about whether other holds a file or data) + // hard to warn for these however ... perhaps just remove early in 0.5.0 ? + "keyFiles", "publicKey", "privateKey"); //TODO remove once everything is prefixed SSHCONFIG_PREFIX or included above public static final Collection<String> NON_SSH_PROPS = ImmutableSet.of("latitude", "longitude", "backup", "sshPublicKeyData", "sshPrivateKeyData"); http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/31b5c079/core/src/main/java/brooklyn/location/basic/jclouds/JcloudsLocation.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/location/basic/jclouds/JcloudsLocation.java b/core/src/main/java/brooklyn/location/basic/jclouds/JcloudsLocation.java index c17c8e0..b886e07 100644 --- a/core/src/main/java/brooklyn/location/basic/jclouds/JcloudsLocation.java +++ b/core/src/main/java/brooklyn/location/basic/jclouds/JcloudsLocation.java @@ -11,6 +11,7 @@ import static org.jclouds.scriptbuilder.domain.Statements.exec; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashMap; @@ -46,7 +47,7 @@ import org.jclouds.openstack.nova.v2_0.compute.options.NovaTemplateOptions; import org.jclouds.scriptbuilder.domain.InterpretableStatement; import org.jclouds.scriptbuilder.domain.Statement; import org.jclouds.scriptbuilder.domain.Statements; -import org.jclouds.scriptbuilder.statements.login.UserAdd; +import org.jclouds.scriptbuilder.statements.login.AdminAccess; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,10 +57,13 @@ import brooklyn.location.NoMachinesAvailableException; import brooklyn.location.basic.AbstractLocation; import brooklyn.location.basic.SshMachineLocation; import brooklyn.location.basic.jclouds.templates.PortableTemplateBuilder; +import brooklyn.util.KeyValueParser; import brooklyn.util.MutableMap; +import brooklyn.util.Time; import brooklyn.util.flags.TypeCoercions; import brooklyn.util.internal.Repeater; import brooklyn.util.text.Identifiers; +import brooklyn.util.text.Strings; import com.google.common.base.Charsets; import com.google.common.base.Throwables; @@ -76,18 +80,26 @@ import com.google.common.util.concurrent.ListenableFuture; * For provisioning and managing VMs in a particular provider/region, using jclouds. * * Configuration flags include the following: - * - userName (defaults to "root") + * - provider (e.g. "aws-ec2") + * - providerLocationId (e.g. "eu-west-1") + * - defaultImageId + * + * - user (defaults to "root" or other known superuser) * - publicKeyFile * - privateKeyFile + * - privateKeyPasspharse + * + * - loginUser (if should initially login as someone other that root / default VM superuser) + * - loginUser.privateKeyFile + * - loginUser.privateKeyPasspharse + * + * // deprecated * - sshPublicKey * - sshPrivateKey * - rootSshPrivateKey (@Beta) * - rootSshPublicKey (@Beta) * - rootSshPublicKeyData (@Beta; calls templateOptions.authorizePublicKey()) * - dontCreateUser (otherwise if user != root, then creates this user) - * - provider (e.g. "aws-ec2") - * - providerLocationId (e.g. "eu-west-1") - * - defaultImageId * * The flags can also includes values passed straight through to jclouds; to the TemplateBuilder: * - minRam @@ -121,6 +133,7 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio * where root@ is not allowed to log in */ public static final List<String> ROOT_ALIASES = ImmutableList.of("ubuntu", "ec2-user"); public static final List<String> NON_ADDABLE_USERS = ImmutableList.<String>builder().add(ROOT_USERNAME).addAll(ROOT_ALIASES).build(); + public static final int START_SSHABLE_TIMEOUT = 5*60*1000; private final Map<String,Map<String, ? extends Object>> tagMapping = Maps.newLinkedHashMap(); @@ -199,45 +212,141 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio useConfig(instance.getConf()); } + @SuppressWarnings({ "unchecked", "rawtypes" }) BrooklynJcloudsSetupHolder useConfig(Map flags) { allconf.putAll(flags); unusedConf.putAll(flags); return this; } + + private Object get(String key) { + unusedConf.remove(key); + return allconf.get(key); + } + private boolean use(String key) { + unusedConf.remove(key); + return truth(allconf.get(key)); + } + + @SuppressWarnings("unchecked") + private boolean setKeyFromKey(String targetKey, String key) { + if (!use(key)) return false; + Object value = get(key); + allconf.put(targetKey, value); + return true; + } + + @SuppressWarnings("unchecked") + private <T> T setKeyToValue(String key, T value) { + allconf.put(key, value); + return value; + } + + String provider; + String providerLocationId; + + String user; + String privateKeyData, privateKeyPassphrase, publicKeyData, password; + /** @deprecated */ + String sshPublicKeyData, sshPrivateKeyData; + + String loginUser; + String loginUser_privateKeyData, loginUser_privateKeyPassphrase, loginUser_password; + BrooklynJcloudsSetupHolder apply() { try { - if (truth(unusedConf.remove("callerContext"))) _callerContext = allconf.get("callerContext"); + if (use("provider")) + provider = ""+get("provider"); + if (use("providerLocationId")) + providerLocationId = ""+get("providerLocationId"); - // this _creates_ the indicated userName (not a good API...) - if (!truth(unusedConf.remove("userName"))) allconf.put("userName", ROOT_USERNAME); + if (use("callerContext")) + _callerContext = get("callerContext"); + + // align user (brooklyn standard) and userName (jclouds standard) fields + // TODO don't default to root --> default later to what jclouds says + if (!use("user") && use("userName")) + setKeyFromKey("user", "userName"); + if (use("user")) + user = (String) get("user"); // perhaps deprecate supply of data (and of different root key?) to keep it simpler? - if (truth(unusedConf.remove("publicKeyFile"))) - allconf.put("sshPublicKeyData", Files.toString(instance.getPublicKeyFile(allconf), Charsets.UTF_8)); - if (truth(unusedConf.remove("privateKeyFile"))) - allconf.put("sshPrivateKeyData", Files.toString(instance.getPrivateKeyFile(allconf), Charsets.UTF_8)); - if (truth(unusedConf.remove("sshPublicKey"))) - allconf.put("sshPublicKeyData", Files.toString(asFile(allconf.get("sshPublicKey")), Charsets.UTF_8)); - if (truth(unusedConf.remove("sshPrivateKey"))) - allconf.put("sshPrivateKeyData", Files.toString(asFile(allconf.get("sshPrivateKey")), Charsets.UTF_8)); - if (truth(unusedConf.remove("rootSshPrivateKey"))) - allconf.put("rootSshPrivateKeyData", Files.toString(asFile(allconf.get("rootSshPrivateKey")), Charsets.UTF_8)); - if (truth(unusedConf.remove("rootSshPublicKey"))) - allconf.put("rootSshPublicKeyData", Files.toString(asFile(allconf.get("rootSshPublicKey")), Charsets.UTF_8)); - if (truth(unusedConf.remove("dontCreateUser"))) - allconf.put("dontCreateUser", true); + if (truth(unusedConf.remove("publicKeyData"))) + publicKeyData = sshPublicKeyData = ""+get("publicKeyData"); + if (truth(unusedConf.remove("privateKeyData"))) + privateKeyData = sshPrivateKeyData = ""+get("privateKeyData"); + if (use("privateKeyFile")) { + if (truth(privateKeyData)) LOG.warn("privateKeyData and privateKeyFile both specified; preferring the former"); + else + privateKeyData = sshPublicKeyData = setKeyToValue("sshPrivateKeyData", Files.toString(instance.getPrivateKeyFile(allconf), Charsets.UTF_8)); + } + if (truth(unusedConf.remove("publicKeyFile"))) { + if (truth(publicKeyData)) LOG.warn("publicKeyData and publicKeyFile both specified; preferring the former"); + else + publicKeyData = sshPublicKeyData = setKeyToValue("sshPublicKeyData", Files.toString(instance.getPublicKeyFile(allconf), Charsets.UTF_8)); + } else if (!truth(publicKeyData) && truth(get("privateKeyFile"))) { + File f = new File(""+get("privateKeyFile")+".pub"); + if (f.exists()) { + LOG.debug("Loading publicKeyData from privateKeyFile + .pub"); + publicKeyData = sshPublicKeyData = setKeyToValue("sshPublicKeyData", Files.toString(f, Charsets.UTF_8)); + } + } + // deprecated: + if (use("sshPublicKey")) + publicKeyData = sshPublicKeyData = setKeyToValue("sshPublicKeyData", Files.toString(asFile(allconf.get("sshPublicKey")), Charsets.UTF_8)); + if (use("sshPrivateKey")) + privateKeyData = sshPrivateKeyData = setKeyToValue("sshPrivateKeyData", Files.toString(asFile(allconf.get("sshPrivateKey")), Charsets.UTF_8)); + + // are these two ever used: + if (use("rootSshPrivateKey")) { + LOG.warn("Using deprecated property rootSshPrivateKey; use loginUser{,.privateKeyFile,...} instead"); + setKeyToValue("rootSshPrivateKeyData", Files.toString(asFile(allconf.get("rootSshPrivateKey")), Charsets.UTF_8)); + } + if (use("rootSshPublicKey")) { + LOG.warn("Using deprecated property rootSshPublicKey; use loginUser{,.publicKeyFile} instead (though public key often not needed)"); + setKeyToValue("rootSshPublicKeyData", Files.toString(asFile(allconf.get("rootSshPublicKey")), Charsets.UTF_8)); + } + // above replaced with below + if (use("loginUser")) { + loginUser = ""+get("loginUser"); + if (use("loginUser.privateKeyData")) { + loginUser_privateKeyData = ""+get("loginUser.privateKeyData"); + } + if (use("loginUser.privateKeyFile")) { + if (loginUser_privateKeyData!=null) + LOG.warn("loginUser private key data and private key file specified; preferring from file"); + loginUser_privateKeyData = setKeyToValue("loginUser.privateKeyData", Files.toString(asFile(allconf.get("loginUser.privateKeyFile")), Charsets.UTF_8)); + } + if (use("loginUser.privateKeyPassphrase")) { + LOG.warn("loginUser.privateKeyPassphrase not supported by jclouds; use a key which does not have a passphrase for the loginUser"); + loginUser_privateKeyPassphrase = ""+get("loginUser.privateKeyPassphrase"); + } + if (use("loginUser.password")) { + loginUser_password = ""+get("loginUser.password"); + } + // these we ignore + use("loginUser.publicKeyData"); + use("loginUser.publicKeyFile"); + + if (loginUser.equals(user)) { + LOG.debug("Detected that jclouds loginUser is the same as regular user; we don't create this user"); + } + } + + if (use("dontCreateUser")) { + LOG.warn("Using deprecated property dontCreateUser; use login.user instead, set equal to the user to run as"); + setKeyToValue("dontCreateUser", true); + } // allows specifying a LoginCredentials object, for use by jclouds, if known for the VM (ie it is non-standard); - if (truth(unusedConf.remove("customCredentials"))) - customCredentials = (LoginCredentials) allconf.get("customCredentials"); + if (use("customCredentials")) + customCredentials = (LoginCredentials) get("customCredentials"); // following values are copies pass-through, no change - unusedConf.remove("privateKeyPassphrase"); - unusedConf.remove("password"); + use("privateKeyPassphrase"); + use("password"); + use("noDefaultSshKeys"); - unusedConf.remove("provider"); - unusedConf.remove("providerLocationId"); - unusedConf.remove("noDefaultSshKeys"); return this; } catch (IOException e) { throw Throwables.propagate(e); @@ -258,7 +367,41 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio if (truth(_callerContext)) return _callerContext.toString(); return "thread "+Thread.currentThread().getId(); } - + + public String setUser(String u) { + String oldUser = user; + user = u; + allconf.put("user", u); + allconf.put("userName", u); + return oldUser; + } + + public String setPassword(String password) { + String oldPassword = this.password; + this.password = password; + allconf.put("password", password); + return oldPassword; + } + + public String setPrivateKeyData(String privateKeyData) { + String oldPrivateKeyData = this.privateKeyData; + this.privateKeyData = privateKeyData; + allconf.put("sshPrivateKeyData", privateKeyData); + return oldPrivateKeyData; + } + + public void set(String key, String value) { + allconf.put(key, value); + } + + public boolean isDontCreateUser() { + if (!use("dontCreateUser")) return false; + Object v = get("dontCreateUser"); + if (v==null) return false; + if (v instanceof Boolean) return ((Boolean)v).booleanValue(); + if (v instanceof CharSequence) return Boolean.parseBoolean(((CharSequence)v).toString()); + throw new IllegalArgumentException("dontCreateUser does not accept value '"+v+"' of type "+v.getClass()); + } } public static final Set<String> getAllSupportedProperties() { @@ -273,9 +416,14 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio //also, we need a way to define imageId (and others?) with a specific location public static final Collection<String> SUPPORTED_BASIC_PROPERTIES = ImmutableSet.of( - "provider", "identity", "credential", "userName", "publicKeyFile", "privateKeyFile", "privateKeyPassphrase", - "sshPublicKey", "sshPrivateKey", "rootSshPrivateKey", "rootSshPublicKey", "groupId", - "providerLocationId", "provider"); + "provider", "identity", "credential", "groupId", "providerLocationId", + "userName", "user", + "publicKeyFile", "privateKeyFile", "publicKeyData", "privateKeyData", "privateKeyPassphrase", + "loginUser", "loginUser.password", "loginUser.publicKeyFile", "loginUser.privateKeyFile", "loginUser.publicKeyData", "loginUser.privateKeyData", "loginUser.privateKeyPassphrase", + // deprecated: + "sshPublicKey", "sshPrivateKey", + "rootSshPrivateKey", "rootSshPublicKey" + ); /** returns public key file, if one has been configured */ public File getPublicKeyFile() { return getPublicKeyFile(getConf()); } @@ -326,10 +474,9 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio NodeMetadata node = null; try { LOG.info("Creating VM in "+ - elvis(setup.allconf.get("providerLocationId"), setup.allconf.get("provider"))+ - " for "+setup.getCallerContext()); + elvis(setup.providerLocationId, setup.provider)+" for "+setup.getCallerContext()); - Template template = buildTemplate(computeService, (String)setup.allconf.get("providerLocationId"), setup); + Template template = buildTemplate(computeService, setup); setup.warnIfUnused("JcloudsLocation.obtain"); @@ -337,50 +484,71 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio node = Iterables.getOnlyElement(nodes, null); LOG.debug("jclouds created {} for {}", node, setup.getCallerContext()); if (node == null) { - throw new IllegalStateException("No nodes returned by jclouds create-nodes in location " - +setup.allconf.get("providerLocationId")+" for "+setup.getCallerContext()); + throw new IllegalStateException("No nodes returned by jclouds create-nodes in " + +setup.provider+"/"+setup.providerLocationId+" for "+setup.getCallerContext()); } LoginCredentials expectedCredentials = setup.customCredentials; if (expectedCredentials!=null) { //set userName and other data, from these credentials - Object oldUsername = setup.allconf.put("userName", expectedCredentials.getUser()); + Object oldUsername = setup.setUser(expectedCredentials.getUser()); LOG.debug("node {} username {} / {} (customCredentials)", new Object[] { node, expectedCredentials.getUser(), oldUsername }); - if (truth(expectedCredentials.getPassword())) setup.allconf.put("password", expectedCredentials.getPassword()); - if (truth(expectedCredentials.getPrivateKey())) setup.allconf.put("sshPrivateKeyData", expectedCredentials.getPrivateKey()); + if (truth(expectedCredentials.getPassword())) setup.setPassword(expectedCredentials.getPassword()); + if (truth(expectedCredentials.getPrivateKey())) setup.setPrivateKeyData(expectedCredentials.getPrivateKey()); } - if (expectedCredentials==null && truth(setup.allconf.get("sshPrivateKeyData"))) { + if (expectedCredentials==null) { expectedCredentials = LoginCredentials.fromCredentials(node.getCredentials()); - String userName = (String) setup.allconf.get("userName"); - LOG.debug("node {} username {} / {} (jclouds)", new Object[] { node, userName, expectedCredentials.getUser() }); + String user = setup.user; + LOG.debug("node {} username {} / {} (jclouds)", new Object[] { node, user, expectedCredentials.getUser() }); if (truth(expectedCredentials.getUser())) { - if ("root".equals(userName) && ROOT_ALIASES.contains(expectedCredentials.getUser())) { - // FIXME should use 'null' as username then learn it from jclouds - // (or use AdminAccess!) - LOG.debug("overriding username 'root' in favour of '"+expectedCredentials.getUser()+"' at {}", node); - setup.allconf.put("userName", expectedCredentials.getUser()); - userName = expectedCredentials.getUser(); + if (user==null) { + setup.setUser(user = expectedCredentials.getUser()); + } else if ("root".equals(user) && ROOT_ALIASES.contains(expectedCredentials.getUser())) { + // deprecated, we used to default username to 'root'; now we leave null, then use autodetected credentials if no user specified + // + LOG.warn("overriding username 'root' in favour of '"+expectedCredentials.getUser()+"' at {}; this behaviour may be removed in future", node); + setup.setUser(user = expectedCredentials.getUser()); } } - expectedCredentials = LoginCredentials.fromCredentials(new Credentials(userName, (String)setup.allconf.get("sshPrivateKeyData"))); //override credentials + String pkd = elvis(setup.privateKeyData, expectedCredentials.getPrivateKey()); + String pwd = elvis(setup.password, expectedCredentials.getPassword()); + if (user==null || (pkd==null && pwd==null)) { + String missing = (user==null ? "user" : "credential"); + LOG.warn("Not able to determine "+missing+" for "+this+" at "+node+"; will likely fail subsequently"); + expectedCredentials = null; + } else { + LoginCredentials.Builder expectedCredentialsBuilder = LoginCredentials.builder(). + user(user); + if (pkd!=null) expectedCredentialsBuilder.noPassword().privateKey(pkd); + else expectedCredentialsBuilder.noPrivateKey().password(pwd); + expectedCredentials = expectedCredentialsBuilder.build(); +// LoginCredentials.fromCredentials(new Credentials(user, pkd!=null ? pkd : pwd)); + } } if (expectedCredentials != null) node = NodeMetadataBuilder.fromNodeMetadata(node).credentials(expectedCredentials).build(); else + // only happens if something broke above... expectedCredentials = LoginCredentials.fromCredentials(node.getCredentials()); // Wait for the VM to be reachable over SSH - if (setup.allconf.get("waitForSshable") != null ? truth(setup.allconf.get("waitForSshable")) : true) { + if (setup.get("waitForSshable") != null ? truth(setup.get("waitForSshable")) : true) { String vmIp = JcloudsUtil.getFirstReachableAddress(node); final NodeMetadata nodeRef = node; final LoginCredentials expectedCredentialsRef = expectedCredentials; + long delayMs = -1; + try { + delayMs = Time.parseTimeString(""+setup.get("waitForSshable")); + } catch (Exception e) {} + if (delayMs<=0) delayMs = START_SSHABLE_TIMEOUT; + LOG.info("Started VM in {} for {}; waiting for it to be sshable on {}@{}", new Object[] { - elvis(setup.allconf.get("providerLocationId"), setup.allconf.get("provider")), + elvis(setup.get("providerLocationId"), setup.get("provider")), setup.getCallerContext(), - setup.allconf.get("userName"), + setup.user, vmIp }); boolean reachable = new Repeater() @@ -389,46 +557,29 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio .until(new Callable<Boolean>() { public Boolean call() { Statement statement = Statements.newStatementList(exec("hostname")); + // NB this assumes passwordless sudo ! ExecResponse response = computeService.runScriptOnNode(nodeRef.getId(), statement, overrideLoginCredentials(expectedCredentialsRef)); - return response.getExitCode() == 0; + return response.getExitStatus() == 0; }}) - .limitTimeTo(START_SSHABLE_TIMEOUT,MILLISECONDS) + .limitTimeTo(delayMs, MILLISECONDS) .run(); if (!reachable) { throw new IllegalStateException("SSH failed for "+ - setup.allconf.get("userName")+"@"+vmIp+" (for "+setup.getCallerContext()+") after waiting "+ - START_SSHABLE_TIMEOUT+"ms"); + setup.user+"@"+vmIp+" (for "+setup.getCallerContext()+") after waiting "+ + Time.makeTimeString(delayMs)); } } - String vmHostname = getPublicHostname(node, setup.allconf); + String vmHostname = getPublicHostname(node, setup); - Map sshConfig = Maps.newLinkedHashMap(); + Map sshConfig = generateSshConfig(setup, node); - if (truth(getPrivateKeyFile())) sshConfig.put("keyFiles", ImmutableList.of(getPrivateKeyFile().getCanonicalPath())); - if (truth(setup.allconf.get("sshPrivateKeyData"))) { - sshConfig.put("privateKey", setup.allconf.get("sshPrivateKeyData")); - sshConfig.put("privateKeyData", setup.allconf.get("sshPrivateKeyData")); - sshConfig.put("sshPrivateKeyData", setup.allconf.get("sshPrivateKeyData")); - } else if (truth(getPrivateKeyFile())) { - sshConfig.put("keyFiles", ImmutableList.of(getPrivateKeyFile().getCanonicalPath())); - } else if (node.getCredentials().getPassword() != null) { - sshConfig.put("password", node.getCredentials().getPassword()); - } - if (truth(setup.allconf.get("privateKeyPassphrase"))) { - // not sure jclouds supports this, but we try, and our ssh routines should use it - sshConfig.put("privateKeyPassphrase", setup.allconf.get("privateKeyPassphrase")); - } - if (truth(setup.allconf.get("sshPublicKeyData"))) { - sshConfig.put("sshPublicKeyData", setup.allconf.get("sshPublicKeyData")); - } - if (LOG.isDebugEnabled()) LOG.debug("creating JcloudsSshMachineLocation for {}@{} for {} with {}", new Object[] { - setup.allconf.get("userName"), + setup.user, vmHostname, setup.getCallerContext(), Entities.sanitize(sshConfig) @@ -437,18 +588,12 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio MutableMap.builder() .put("address", vmHostname) .put("displayName", vmHostname) - .put("user", setup.allconf.get("userName")) + .put("user", setup.user) .put("config", sshConfig) .build(), this, node); - if (truth(setup.allconf.get("sshPrivateKeyData"))) - sshLocByHostname.configure(MutableMap.of("sshPrivateKeyData", setup.allconf.get("sshPrivateKeyData"))); - if (truth(setup.allconf.get("sshPublicKeyData"))) - sshLocByHostname.configure(MutableMap.of("sshPublicKeyData", setup.allconf.get("sshPublicKeyData"))); - if (truth(setup.allconf.get("password"))) - sshLocByHostname.configure(MutableMap.of("password", setup.allconf.get("password"))); - + sshLocByHostname.setParentLocation(this); vmInstanceIds.put(sshLocByHostname, node.getId()); @@ -475,61 +620,73 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio } public JcloudsSshMachineLocation rebindMachine(Map flags, NodeMetadata metadata) throws NoMachinesAvailableException { BrooklynJcloudsSetupHolder setup = new BrooklynJcloudsSetupHolder(this).useConfig(flags).apply(); + if (!setup.use("id")) setup.set("id", metadata.getId()); + setHostnameUpdatingCredentials(setup, metadata); + return rebindMachine(setup.allconf); + } + + protected void setHostnameUpdatingCredentials(BrooklynJcloudsSetupHolder setup, NodeMetadata metadata) { + List<String> usersTried = new ArrayList<String>(); + + if (truth(setup.user)) { + if (setHostname(setup, metadata, false)) return; + usersTried.add(setup.user); + } - Map newFlags = setup.allconf; - newFlags.put("id", metadata.getId()); LoginCredentials credentials = metadata.getCredentials(); if (truth(credentials)) { - if (truth(credentials.getUser())) newFlags.put("userName", credentials.getUser()); - if (truth(credentials.getPrivateKey())) newFlags.put("sshPrivateKeyData", credentials.getPrivateKey()); - } else { - //username should already be set - if (!truth(newFlags.get("privateKeyFile"))) - newFlags.put("privateKeyFile", getPrivateKeyFile()); + if (truth(credentials.getUser())) setup.setUser(credentials.getUser()); + if (truth(credentials.getPrivateKey())) setup.setPrivateKeyData(credentials.getPrivateKey()); + if (setHostname(setup, metadata, false)) return; + usersTried.add(setup.user); + } + + for (String u: NON_ADDABLE_USERS) { + setup.setUser(u); + if (setHostname(setup, metadata, false)) { + LOG.warn("Auto-detected user at "+metadata+" as "+setup.user+" (other attempted users "+usersTried+" cannot access it)"); + return; + } + usersTried.add(setup.user); } + // just repeat, so we throw exception + LOG.warn("Failed to log in to "+metadata+", tried as users "+usersTried+" (throwing original exception)"); + setHostname(setup, metadata, true); + } + + protected boolean setHostname(BrooklynJcloudsSetupHolder setup, NodeMetadata metadata, boolean rethrow) { try { - newFlags.put("hostname", getPublicHostname(metadata, newFlags)); + setup.set("hostname", getPublicHostname(metadata, setup)); + return true; } catch (Exception e) { - // TODO this logic should be placed somewhere more useful/shared - //try again with user ubuntu, then root - newFlags.put("userName", "ubuntu"); - try { - newFlags.put("hostname", getPublicHostname(metadata, newFlags)); - } catch (Exception e2) { - newFlags.put("userName", "root"); - try { - newFlags.put("hostname", getPublicHostname(metadata, newFlags)); - } catch (Exception e3) { - LOG.warn("couldn't access "+metadata+" to discover hostname (rethrowing): "+e); - throw Throwables.propagate(e); - } + if (rethrow) { + LOG.warn("couldn't connect to "+metadata+" when trying to discover hostname (rethrowing): "+e); + throw Throwables.propagate(e); } - LOG.info("remapping username at "+metadata+" to "+newFlags.get("userName")+" (this username works)"); + return false; } - return rebindMachine(newFlags); } - + /** * Brings an existing machine with the given details under management. * <p> * Required fields are: * <ul> - * <li>id: the jclouds VM id, e.g. "eu-west-1/i-5504f21d" + * <li>id: the jclouds VM id, e.g. "eu-west-1/i-5504f21d" (NB this is @see JcloudsSshMachineLocation#getJcloudsId() not #getId()) * <li>hostname: the public hostname or IP of the machine, e.g. "ec2-176-34-93-58.eu-west-1.compute.amazonaws.com" * <li>userName: the username for ssh'ing into the machine * <ul> */ public JcloudsSshMachineLocation rebindMachine(Map flags) throws NoMachinesAvailableException { try { - String id = (String) checkNotNull(flags.get("id"), "id"); - String hostname = (String) checkNotNull(flags.get("hostname"), "hostname"); - String username = (String) checkNotNull(flags.get("userName"), "userName"); - String password = (String) flags.get("password"); + BrooklynJcloudsSetupHolder setup = new BrooklynJcloudsSetupHolder(this).useConfig(flags).apply(); + String id = (String) checkNotNull(setup.get("id"), "id"); + String hostname = (String) checkNotNull(setup.get("hostname"), "hostname"); + String user = (String) checkNotNull(setup.user, "user"); LOG.info("Rebinding to VM {} ({}@{}), in jclouds location for provider {}", - new Object[] {id, username, hostname, getProvider()}); + new Object[] {id, user, hostname, getProvider()}); - BrooklynJcloudsSetupHolder setup = new BrooklynJcloudsSetupHolder(this).useConfig(flags).apply(); ComputeService computeService = JcloudsUtil.buildComputeService(setup.allconf, setup.unusedConf); NodeMetadata node = computeService.getNodeMetadata(id); @@ -537,47 +694,25 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio throw new IllegalArgumentException("Node not found with id "+id); } - if (truth(setup.allconf.get("sshPrivateKeyData"))) { - LoginCredentials expectedCredentials = LoginCredentials.fromCredentials(new Credentials((String)setup.allconf.get("userName"), (String)setup.allconf.get("sshPrivateKeyData"))); + if (truth(setup.privateKeyData)) { + LoginCredentials expectedCredentials = LoginCredentials.fromCredentials(new Credentials(setup.user, setup.privateKeyData)); //override credentials node = NodeMetadataBuilder.fromNodeMetadata(node).credentials(expectedCredentials).build(); } // TODO confirm we can SSH ? - - Map sshConfig = Maps.newLinkedHashMap(); - if (password != null) { - sshConfig.put("password", password); - } else { - if (truth(getPrivateKeyFile())) sshConfig.put("keyFiles", ImmutableList.of(getPrivateKeyFile().getCanonicalPath())); - if (truth(setup.allconf.get("sshPrivateKeyData"))) { - sshConfig.put("privateKey", setup.allconf.get("sshPrivateKeyData")); - sshConfig.put("privateKeyData", setup.allconf.get("sshPrivateKeyData")); - sshConfig.put("sshPrivateKeyData", setup.allconf.get("sshPrivateKeyData")); - } else if (truth(getPrivateKeyFile())) { - sshConfig.put("keyFiles", ImmutableList.of(getPrivateKeyFile().getCanonicalPath())); - } else if (node.getCredentials().getPassword() != null) { - sshConfig.put("password", node.getCredentials().getPassword()); - } - if (truth(setup.allconf.get("sshPublicKeyData"))) { - sshConfig.put("sshPublicKeyData", setup.allconf.get("sshPublicKeyData")); - } - } + + Map sshConfig = generateSshConfig(setup, node); JcloudsSshMachineLocation sshLocByHostname = new JcloudsSshMachineLocation( MutableMap.builder() .put("address", hostname) .put("displayName", hostname) - .put("user", username) + .put("user", user) .put("config", sshConfig) .build(), this, node); - - if (truth(setup.allconf.get("sshPrivateKeyData"))) - sshLocByHostname.configure(MutableMap.of("sshPrivateKeyData", setup.allconf.get("sshPrivateKeyData"))); - if (truth(setup.allconf.get("sshPublicKeyData"))) - sshLocByHostname.configure(MutableMap.of("sshPublicKeyData", setup.allconf.get("sshPublicKeyData"))); - + sshLocByHostname.setParentLocation(this); vmInstanceIds.put(sshLocByHostname, node.getId()); @@ -586,12 +721,48 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio throw Throwables.propagate(e); } } + + private Map generateSshConfig(BrooklynJcloudsSetupHolder setup, NodeMetadata node) throws IOException { + Map sshConfig = Maps.newLinkedHashMap(); + if (truth(setup.allconf.get("sshPrivateKeyData"))) { + sshConfig.put("privateKey", setup.privateKeyData); + // sshConfig.put("privateKeyData", setup.allconf.get("sshPrivateKeyData")); + // sshConfig.put("sshPrivateKeyData", setup.allconf.get("sshPrivateKeyData")); + } else if (truth(getPrivateKeyFile())) { + sshConfig.put("keyFiles", ImmutableList.of(getPrivateKeyFile().getCanonicalPath())); + } else if (truth(getPrivateKeyFile())) { + sshConfig.put("keyFiles", ImmutableList.of(getPrivateKeyFile().getCanonicalPath())); + } else if (setup.password != null) { + sshConfig.put("password", setup.password); + } else if (node!=null && node.getCredentials().getPassword() != null) { + sshConfig.put("password", node.getCredentials().getPassword()); + } + + if (truth(setup.allconf.get("privateKeyPassphrase"))) { + // TODO do we set this up correctly for jclouds to use? + sshConfig.put("privateKeyPassphrase", setup.privateKeyPassphrase); + } + +// if (truth(setup.allconf.get("sshPublicKeyData"))) { +// sshConfig.put("sshPublicKeyData", setup.allconf.get("sshPublicKeyData")); +// } + +// if (truth(setup.allconf.get("sshPrivateKeyData"))) +// sshLocByHostname.configure(MutableMap.of("sshPrivateKeyData", setup.allconf.get("sshPrivateKeyData"))); +// if (truth(setup.allconf.get("sshPublicKeyData"))) +// sshLocByHostname.configure(MutableMap.of("sshPublicKeyData", setup.allconf.get("sshPublicKeyData"))); +// if (truth(setup.allconf.get("password"))) +// sshLocByHostname.configure(MutableMap.of("password", setup.allconf.get("password"))); + + return sshConfig; + } public static String generateGroupId() { // In jclouds 1.5, there are strict rules for group id: it must be DNS compliant, and no more than 15 characters + // TODO surely this can be overridden! it's so silly being so short in common places ... or at least set better metadata? String user = System.getProperty("user.name"); - String rand = Identifiers.makeRandomId(2); - String result = "br-"+(user.substring(0,Math.min(user.length(),5)))+"-"+rand; + String rand = Identifiers.makeRandomId(6); + String result = "br-"+Strings.maxlen(user, 4)+"-"+rand; return result.toLowerCase(); } @@ -745,40 +916,51 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio public void apply(TemplateOptions t, Map props, Object v) { t.runAsRoot((Boolean)v); }}) + .put("loginUser", new CustomizeTemplateOptions() { + public void apply(TemplateOptions t, Map props, Object v) { + t.overrideLoginUser(((CharSequence)v).toString()); + }}) + .put("loginUser.password", new CustomizeTemplateOptions() { + public void apply(TemplateOptions t, Map props, Object v) { + t.overrideLoginPassword(((CharSequence)v).toString()); + }}) + .put("loginUser.privateKeyData", new CustomizeTemplateOptions() { + public void apply(TemplateOptions t, Map props, Object v) { + t.overrideLoginPrivateKey(((CharSequence)v).toString()); + }}) .put("overrideLoginUser", new CustomizeTemplateOptions() { public void apply(TemplateOptions t, Map props, Object v) { + LOG.warn("Using deprecated property overrideLoginUser; use loginUser instead"); t.overrideLoginUser(((CharSequence)v).toString()); }}) .build(); private static boolean listedAvailableTemplatesOnNoSuchTemplate = false; - private Template buildTemplate(ComputeService computeService, String providerLocationId, BrooklynJcloudsSetupHolder setup) { - Map<String,? extends Object> properties = setup.allconf; - Map unusedConf = setup.unusedConf; - TemplateBuilder templateBuilder = (TemplateBuilder) unusedConf.remove("templateBuilder"); + private Template buildTemplate(ComputeService computeService, BrooklynJcloudsSetupHolder setup) { + TemplateBuilder templateBuilder = (TemplateBuilder) setup.get("templateBuilder"); if (templateBuilder==null) templateBuilder = new PortableTemplateBuilder(); else LOG.debug("jclouds using templateBuilder {} as base for provisioning in {} for {}", new Object[] {templateBuilder, this, setup.getCallerContext()}); - if (providerLocationId!=null) { - templateBuilder.locationId(providerLocationId); + if (setup.providerLocationId!=null) { + templateBuilder.locationId(setup.providerLocationId); } for (Map.Entry<String, CustomizeTemplateBuilder> entry : SUPPORTED_TEMPLATE_BUILDER_PROPERTIES.entrySet()) { String name = entry.getKey(); CustomizeTemplateBuilder code = entry.getValue(); - if (unusedConf.remove(name)!=null) - code.apply(templateBuilder, properties, properties.get(name)); + if (setup.use(name)) + code.apply(templateBuilder, setup.allconf, setup.get(name)); } if (templateBuilder instanceof PortableTemplateBuilder) { ((PortableTemplateBuilder)templateBuilder).attachComputeService(computeService); // do the default last, and only if nothing else specified (guaranteed to be a PTB if nothing else specified) - if (truth(unusedConf.remove("defaultImageId"))) { + if (setup.use("defaultImageId")) { if (((PortableTemplateBuilder)templateBuilder).isBlank()) { - CharSequence defaultImageId = (CharSequence) properties.get("defaultImageId"); + CharSequence defaultImageId = (CharSequence) setup.get("defaultImageId"); templateBuilder.imageId(defaultImageId.toString()); } } @@ -830,35 +1012,46 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio for (Map.Entry<String, CustomizeTemplateOptions> entry : SUPPORTED_TEMPLATE_OPTIONS_PROPERTIES.entrySet()) { String name = entry.getKey(); CustomizeTemplateOptions code = entry.getValue(); - if (unusedConf.remove(name)!=null) - code.apply(options, properties, properties.get(name)); + if (setup.use(name)) + code.apply(options, setup.allconf, setup.get(name)); } + + // Setup the user - if (NON_ADDABLE_USERS.contains(properties.get("userName")) && truth(properties.get("sshPublicKeyData"))) { - String keyData = (String) properties.get("sshPublicKeyData"); - options.authorizePublicKey(keyData); - } //NB: we ignore private key here because, by default we probably should not be installing it remotely; //also, it may not be valid for first login (it is created before login e.g. on amazon, so valid there; //but not elsewhere, e.g. on rackspace) - - // Setup the user - if (truth(properties.get("userName")) && !NON_ADDABLE_USERS.contains(properties.get("userName")) && - !truth(properties.get("dontCreateUser"))) { - UserAdd.Builder userBuilder = UserAdd.builder(); - userBuilder.login((String)properties.get("userName")); - String publicKeyData = (String) properties.get("sshPublicKeyData"); - userBuilder.authorizeRSAPublicKey(publicKeyData); - Statement userBuilderStatement = userBuilder.build(); - options.runScript(userBuilderStatement); + if (truth(setup.user) && !NON_ADDABLE_USERS.contains(setup.user) && + !setup.user.equals(setup.loginUser) && !truth(setup.isDontCreateUser())) { + // create the user, if it's not the login user and not a known root-level user + // by default we now give these users sudo privileges. + // if you want something else, that can be specified manually, + // e.g. using jclouds UserAdd.Builder, with RunScriptOnNode, or template.options.runScript(xxx) + // (if that is a common use case, we could expose a property here) + // note AdminAccess requires _all_ fields set, due to http://code.google.com/p/jclouds/issues/detail?id=1095 + AdminAccess.Builder adminBuilder = AdminAccess.builder(). + adminUsername(setup.user). + grantSudoToAdminUser(true); + adminBuilder.adminPassword(setup.use("password") ? setup.password : Identifiers.makeRandomId(12)); + if (setup.publicKeyData!=null) + adminBuilder.authorizeAdminPublicKey(true).adminPublicKey(setup.publicKeyData); + else + adminBuilder.authorizeAdminPublicKey(false).adminPublicKey("ignored").lockSsh(true); + adminBuilder.installAdminPrivateKey(false).adminPrivateKey("ignored"); + adminBuilder.resetLoginPassword(true).loginPassword(Identifiers.makeRandomId(12)); + adminBuilder.lockSsh(true); + options.runScript(adminBuilder.build()); + } else if (truth(setup.publicKeyData)) { + // don't create the user, but authorize the public key for the default user + options.authorizePublicKey(setup.publicKeyData); } - LOG.debug("jclouds using template {} to provision machine in {} for {}", new Object[] {template, this, setup.getCallerContext()}); + LOG.debug("jclouds using template {} / options {} to provision machine in {} for {}", new Object[] {template, options, this, setup.getCallerContext()}); return template; } - private String getPublicHostname(NodeMetadata node, Map allconf) { - if ("aws-ec2".equals(allconf != null ? allconf.get("provider") : null)) { + private String getPublicHostname(NodeMetadata node, BrooklynJcloudsSetupHolder setup) { + if ("aws-ec2".equals(setup != null ? setup.get("provider") : null)) { String vmIp = null; try { vmIp = JcloudsUtil.getFirstReachableAddress(node); @@ -867,7 +1060,7 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio } if (vmIp != null) { try { - return getPublicHostnameAws(vmIp, allconf); + return getPublicHostnameAws(vmIp, setup); } catch (Exception e) { LOG.warn("Error querying aws-ec2 instance over ssh for its hostname; falling back to first reachable IP", e); return vmIp; @@ -875,12 +1068,13 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio } } - return getPublicHostnameGeneric(node, allconf); + return getPublicHostnameGeneric(node, setup); } - private String getPublicHostnameGeneric(NodeMetadata node, @Nullable Map allconf) { + private String getPublicHostnameGeneric(NodeMetadata node, @Nullable BrooklynJcloudsSetupHolder setup) { //prefer the public address to the hostname because hostname is sometimes wrong/abbreviated //(see that javadoc; also e.g. on rackspace, the hostname lacks the domain) + //TODO would it be better to prefer hostname, but first check that it is resolvable? if (truth(node.getPublicAddresses())) { return node.getPublicAddresses().iterator().next(); } else if (truth(node.getHostname())) { @@ -892,19 +1086,21 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio } } - private String getPublicHostnameAws(String ip, Map allconf) { + private String getPublicHostnameAws(String ip, BrooklynJcloudsSetupHolder setup) { try { Map sshConfig = Maps.newLinkedHashMap(); - if (truth(allconf.get("password"))) - sshConfig.put("password", allconf.get("password")); + // TODO combine with above + if (truth(setup.password)) + sshConfig.put("password", setup.password); + if (truth(setup.privateKeyData)) + sshConfig.put("privateKeyData", setup.privateKeyData); + if (truth(setup.privateKeyPassphrase)) + sshConfig.put("privateKeyPassphrase", setup.privateKeyPassphrase); if (truth(getPrivateKeyFile())) - sshConfig.put("keyFiles", ImmutableList.of(getPrivateKeyFile().getCanonicalPath())); - if (truth(allconf.get("sshPrivateKeyData"))) - sshConfig.put("privateKeyData", allconf.get("sshPrivateKeyData")); - if (truth(allconf.get("privateKeyPassphrase"))) - sshConfig.put("privateKeyPassphrase", allconf.get("privateKeyPassphrase")); + sshConfig.put("keyFiles", ImmutableList.of(getPrivateKeyFile().getCanonicalPath())); + // TODO messy way to get an SSH session - SshMachineLocation sshLocByIp = new SshMachineLocation(MutableMap.of("address", ip, "user", allconf.get("userName"), "config", sshConfig)); + SshMachineLocation sshLocByIp = new SshMachineLocation(MutableMap.of("address", ip, "user", setup.user, "config", sshConfig)); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); ByteArrayOutputStream errStream = new ByteArrayOutputStream(); @@ -1079,6 +1275,8 @@ public class JcloudsLocation extends AbstractLocation implements MachineProvisio result.put(key, value); } return result; + } else if (v instanceof CharSequence) { + return KeyValueParser.parseMap(v.toString()); } else { throw new IllegalArgumentException("Invalid type for Map<String,String>: "+v+" of type "+v.getClass()); } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/31b5c079/core/src/main/java/brooklyn/location/basic/jclouds/JcloudsUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/location/basic/jclouds/JcloudsUtil.java b/core/src/main/java/brooklyn/location/basic/jclouds/JcloudsUtil.java index b89d4fa..b019bd2 100644 --- a/core/src/main/java/brooklyn/location/basic/jclouds/JcloudsUtil.java +++ b/core/src/main/java/brooklyn/location/basic/jclouds/JcloudsUtil.java @@ -12,6 +12,7 @@ import static org.jclouds.scriptbuilder.domain.Statements.newStatementList; import java.io.File; import java.io.IOException; import java.net.URI; +import java.util.Arrays; import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; @@ -21,6 +22,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.jclouds.Constants; +import org.jclouds.ContextBuilder; import org.jclouds.aws.ec2.AWSEC2Client; import org.jclouds.compute.ComputeService; import org.jclouds.compute.ComputeServiceContext; @@ -227,7 +229,18 @@ public class JcloudsUtil { new SshjSshClientModule(), new SLF4JLoggingModule(), new BouncyCastleCryptoModule()); - + + // TODO update to new (jclouds 1.5) syntax + // this is the syntax, it's kinda hard to figure out, + // but nice enough once you know it! +// ComputeServiceContext computeServiceContext = ContextBuilder.newBuilder("aws-ec2"). +// modules(Arrays.asList(new SshjSshClientModule(), new SLF4JLoggingModule())). +// credentials(identity, credential). +// overrides(properties). +// build(ComputeServiceContext.class); +// +// final ComputeService computeService = computeServiceContext.getComputeService(); + ComputeServiceContextFactory computeServiceFactory = new ComputeServiceContextFactory(); ComputeService computeService = computeServiceFactory http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/31b5c079/core/src/main/java/brooklyn/location/basic/jclouds/templates/AbstractPortableTemplateBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/location/basic/jclouds/templates/AbstractPortableTemplateBuilder.java b/core/src/main/java/brooklyn/location/basic/jclouds/templates/AbstractPortableTemplateBuilder.java index e8a8023..ffd8121 100644 --- a/core/src/main/java/brooklyn/location/basic/jclouds/templates/AbstractPortableTemplateBuilder.java +++ b/core/src/main/java/brooklyn/location/basic/jclouds/templates/AbstractPortableTemplateBuilder.java @@ -357,7 +357,7 @@ public abstract class AbstractPortableTemplateBuilder<T extends AbstractPortable public boolean isBlank() { if (commands.isEmpty()) return true; //also "blank" if we've blanked it - if (commands.size()==1 && minRam==1) return true; + if (commands.size()==1 && (minRam!=null && minRam==1)) return true; return false; } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/31b5c079/core/src/main/java/brooklyn/util/text/Strings.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/util/text/Strings.java b/core/src/main/java/brooklyn/util/text/Strings.java index de30dd0..7bcecd6 100644 --- a/core/src/main/java/brooklyn/util/text/Strings.java +++ b/core/src/main/java/brooklyn/util/text/Strings.java @@ -481,4 +481,9 @@ public class Strings { return ("a"+s).trim().substring(1); } + /** returns up to maxlen characters from the start of s */ + public static String maxlen(String s, int maxlen) { + return s.substring(0, Math.min(s.length(), maxlen)); + } + } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/31b5c079/core/src/test/java/brooklyn/location/basic/jclouds/JcloudsLocationRebindTest.groovy ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/location/basic/jclouds/JcloudsLocationRebindTest.groovy b/core/src/test/java/brooklyn/location/basic/jclouds/JcloudsLocationRebindTest.groovy index ed7e471..a8ed96b 100644 --- a/core/src/test/java/brooklyn/location/basic/jclouds/JcloudsLocationRebindTest.groovy +++ b/core/src/test/java/brooklyn/location/basic/jclouds/JcloudsLocationRebindTest.groovy @@ -96,10 +96,43 @@ public class JcloudsLocationRebindTest { String id = machine.getJcloudsId() InetAddress address = machine.getAddress() String hostname = address.getHostName() + String user = machine.getUser() + + // Create a new jclouds location, and re-bind the existing VM to that + JcloudsLocation loc2 = locFactory.newLocation(EUWEST_REGION_NAME) + SshMachineLocation machine2 = loc2.rebindMachine(id:id, hostname:hostname, user:user) + + // Confirm the re-bound machine is wired up + assertTrue(machine2.isSshable()) + assertEquals(ImmutableSet.copyOf(loc2.getChildLocations()), ImmutableSet.of(machine2)) + + // Confirm can release the re-bound machine via the new jclouds location + loc2.release(machine2) + assertFalse(machine2.isSshable()) + assertEquals(ImmutableSet.copyOf(loc2.getChildLocations()), Collections.emptySet()) + } + + @Test(groups = [ "Live" ]) + public void testRebindVmDeprecated() { + loc = locFactory.newLocation(EUWEST_REGION_NAME) + loc.setTagMapping([MyEntityType:[ + imageId:EUWEST_IMAGE_ID, + imageOwner:IMAGE_OWNER + ]]) + + // Create a VM through jclouds + Map flags = loc.getProvisioningFlags(["MyEntityType"]) + JcloudsSshMachineLocation machine = obtainMachine(flags) + assertTrue(machine.isSshable()) + + String id = machine.getJcloudsId() + InetAddress address = machine.getAddress() + String hostname = address.getHostName() String username = machine.getUser() // Create a new jclouds location, and re-bind the existing VM to that JcloudsLocation loc2 = locFactory.newLocation(EUWEST_REGION_NAME) + // pass deprecated userName SshMachineLocation machine2 = loc2.rebindMachine(id:id, hostname:hostname, userName:username) // Confirm the re-bound machine is wired up http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/31b5c079/core/src/test/java/brooklyn/location/basic/jclouds/SimpleJcloudsLocationUserLoginAndConfigTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/location/basic/jclouds/SimpleJcloudsLocationUserLoginAndConfigTest.java b/core/src/test/java/brooklyn/location/basic/jclouds/SimpleJcloudsLocationUserLoginAndConfigTest.java new file mode 100644 index 0000000..485b1dd --- /dev/null +++ b/core/src/test/java/brooklyn/location/basic/jclouds/SimpleJcloudsLocationUserLoginAndConfigTest.java @@ -0,0 +1,221 @@ +package brooklyn.location.basic.jclouds; + +import java.io.ByteArrayOutputStream; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.Test; + +import brooklyn.location.NoMachinesAvailableException; +import brooklyn.location.basic.JcloudsResolver; +import brooklyn.location.basic.SshMachineLocation; +import brooklyn.location.basic.jclouds.JcloudsLocation.JcloudsSshMachineLocation; +import brooklyn.test.TestUtils; +import brooklyn.util.MutableMap; +import brooklyn.util.text.Identifiers; + +import com.google.common.base.Throwables; + +public class SimpleJcloudsLocationUserLoginAndConfigTest { + + private static final Logger log = LoggerFactory.getLogger(SimpleJcloudsLocationUserLoginAndConfigTest.class); + + @SuppressWarnings("rawtypes") + @Test(groups="Live") + public void testJcloudsCreateBogStandard() throws Exception { + log.info("TEST testJcloudsCreateBogStandard"); + JcloudsLocation l = JcloudsResolver.resolve("aws-ec2:us-east-1"); + JcloudsSshMachineLocation m1 = l.obtain(); + try { + Map details = MutableMap.of("id", m1.getJcloudsId(), "hostname", m1.getAddress().getHostAddress(), "user", m1.getUser()); + log.info("got machine "+m1+" at "+l+": "+details+"; now trying to rebind"); + String result; + // echo conflates spaces of arguments + result = execWithOutput(m1, Arrays.asList("echo trying m1", "hostname", "date")); + Assert.assertTrue(result.contains("trying m1")); + + log.info("now trying rebind "+m1); + JcloudsSshMachineLocation m2 = l.rebindMachine(details); + result = execWithOutput(m2, Arrays.asList("echo trying m2", "hostname", "date")); + Assert.assertTrue(result.contains("trying m2")); + } finally { + l.release(m1); + } + } + + @SuppressWarnings("rawtypes") + @Test(groups="Live") + public void testJcloudsCreateBogStandardWithUserBrooklyn() throws Exception { + log.info("TEST testJcloudsCreateBogStandardWithUserBrooklyn"); + JcloudsLocation l = JcloudsResolver.resolve("aws-ec2:us-east-1"); + JcloudsSshMachineLocation m1 = l.obtain(MutableMap.of("user", "brooklyn")); + try { + Map details = MutableMap.of("id", m1.getJcloudsId(), "hostname", m1.getAddress().getHostAddress(), "user", m1.getUser()); + log.info("got machine "+m1+" at "+l+": "+details+"; now trying to rebind"); + String result; + // echo conflates spaces of arguments + result = execWithOutput(m1, Arrays.asList("echo trying m1", "hostname", "date")); + Assert.assertTrue(result.contains("trying m1")); + + log.info("now trying rebind "+m1); + JcloudsSshMachineLocation m2 = l.rebindMachine(details); + result = execWithOutput(m2, Arrays.asList("echo trying m2", "hostname", "date")); + Assert.assertTrue(result.contains("trying m2")); + + Assert.assertEquals(m2.getUser(), "brooklyn"); + } finally { + l.release(m1); + } + } + + @SuppressWarnings("rawtypes") + @Test(groups="Live") + public void testJcloudsCreateUserMetadata() throws Exception { + log.info("TEST testJcloudsCreateBogStandard"); + JcloudsLocation l = JcloudsResolver.resolve("aws-ec2:us-east-1"); + String key = "brooklyn-test-user-data"; + String value = "test-"+Identifiers.makeRandomId(4); + JcloudsSshMachineLocation m1 = l.obtain(MutableMap.of("userMetadata", key+"="+value)); + try { + Map details = MutableMap.of("id", m1.getJcloudsId(), "hostname", m1.getAddress().getHostAddress(), "user", m1.getUser(), + "userMetadata", key+"="+value); + Assert.assertEquals(m1.node.getUserMetadata().get(key), value); + + log.info("got machine "+m1+" at "+l+": "+details+"; now trying to rebind"); + String result; + // echo conflates spaces of arguments + result = execWithOutput(m1, Arrays.asList("echo trying m1", "hostname", "date")); + Assert.assertTrue(result.contains("trying m1")); + + log.info("now trying rebind "+m1); + JcloudsSshMachineLocation m2 = l.rebindMachine(details); + result = execWithOutput(m2, Arrays.asList("echo trying m2", "hostname", "date")); + Assert.assertTrue(result.contains("trying m2")); + Assert.assertEquals(m2.node.getUserMetadata().get(key), value); + } finally { + l.release(m1); + } + } + + // a curious image, centos, but user is ec2-user, and handily not correctly auto-detected + // test we can specify a loginUser different from user, and that user is created etc... + // imageId=us-east-1/ami-f95cf390 + public static final String EC2_CENTOS_IMAGE = "us-east-1/ami-f95cf390"; + + @Test(groups="Live") + public void testJcloudsMissingUser() throws Exception { + log.info("TEST testJcloudsMissingUser"); + final JcloudsLocation l = JcloudsResolver.resolve("aws-ec2:us-east-1"); + TestUtils.assertFails(new Runnable() { + public void run() { + try { + // wait up to 30s for login (override default of 5m so test runs faster) + l.obtain(MutableMap.of("imageId", EC2_CENTOS_IMAGE, + "waitForSshable", 30*1000)); + log.info("whoops we logged in"); + } catch (NoMachinesAvailableException e) { + log.info("got error as expected, for missing user: "+e); + Throwables.propagate(e); + } + } + }); + } + + @SuppressWarnings("rawtypes") + @Test(groups="Live") + public void testJcloudsWithSpecificLoginUserAndSameUser() throws Exception { + log.info("TEST testJcloudsWithSpecificLoginUserAndSameUser"); + JcloudsLocation l = JcloudsResolver.resolve("aws-ec2:us-east-1"); + JcloudsSshMachineLocation m1 = l.obtain(MutableMap.of("imageId", EC2_CENTOS_IMAGE, + "loginUser", "ec2-user", + "user", "ec2-user", + "waitForSshable", 30*1000)); + try { + Map details = MutableMap.of("id", m1.getJcloudsId(), "hostname", m1.getAddress().getHostAddress(), "user", m1.getUser()); + log.info("got machine "+m1+" at "+l+": "+details+"; now trying to rebind"); + String result; + // echo conflates spaces of arguments + result = execWithOutput(m1, Arrays.asList("echo trying m1", "hostname", "date")); + Assert.assertTrue(result.contains("trying m1")); + + log.info("now trying rebind "+m1); + JcloudsSshMachineLocation m2 = l.rebindMachine(details); + result = execWithOutput(m2, Arrays.asList("echo trying m2", "hostname", "date")); + Assert.assertTrue(result.contains("trying m2")); + + Assert.assertEquals(m2.getUser(), "ec2-user"); + } finally { + l.release(m1); + } + } + + @SuppressWarnings("rawtypes") + @Test(groups="Live") + public void testJcloudsWithSpecificLoginUserAndNewUser() throws Exception { + log.info("TEST testJcloudsWithSpecificLoginUserAndNewUser"); + JcloudsLocation l = JcloudsResolver.resolve("aws-ec2:us-east-1"); + JcloudsSshMachineLocation m1 = l.obtain(MutableMap.of("imageId", EC2_CENTOS_IMAGE, + "loginUser", "ec2-user", + "user", "newbob", + "waitForSshable", 30*1000)); + try { + Map details = MutableMap.of("id", m1.getJcloudsId(), "hostname", m1.getAddress().getHostAddress(), "user", m1.getUser()); + log.info("got machine "+m1+" at "+l+": "+details+"; now trying to rebind"); + String result; + // echo conflates spaces of arguments + result = execWithOutput(m1, Arrays.asList("echo trying m1", "hostname", "date")); + Assert.assertTrue(result.contains("trying m1")); + + log.info("now trying rebind "+m1); + JcloudsSshMachineLocation m2 = l.rebindMachine(details); + result = execWithOutput(m2, Arrays.asList("echo trying m2", "hostname", "date")); + Assert.assertTrue(result.contains("trying m2")); + + Assert.assertEquals(m2.getUser(), "newbob"); + } finally { + l.release(m1); + } + } + + @SuppressWarnings("rawtypes") + @Test(groups="Live") + public void testJcloudsWithSpecificLoginUserAndDefaultUser() throws Exception { + log.info("TEST testJcloudsWithSpecificLoginUserAndDefaultUser"); + JcloudsLocation l = JcloudsResolver.resolve("aws-ec2:us-east-1"); + JcloudsSshMachineLocation m1 = l.obtain(MutableMap.of("imageId", EC2_CENTOS_IMAGE, + "loginUser", "ec2-user", + "waitForSshable", 30*1000)); + try { + Map details = MutableMap.of("id", m1.getJcloudsId(), "hostname", m1.getAddress().getHostAddress(), "user", m1.getUser()); + log.info("got machine "+m1+" at "+l+": "+details+"; now trying to rebind"); + String result; + // echo conflates spaces of arguments + result = execWithOutput(m1, Arrays.asList("echo trying m1", "hostname", "date")); + Assert.assertTrue(result.contains("trying m1")); + + log.info("now trying rebind "+m1); + JcloudsSshMachineLocation m2 = l.rebindMachine(details); + result = execWithOutput(m2, Arrays.asList("echo trying m2", "hostname", "date")); + Assert.assertTrue(result.contains("trying m2")); + } finally { + l.release(m1); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private String execWithOutput(SshMachineLocation m, List commands) { + Map flags = new LinkedHashMap(); + ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + ByteArrayOutputStream stderr = new ByteArrayOutputStream(); + flags.put("out", stdout); + flags.put("err", stderr); + m.execCommands(flags, "test", commands); + log.info("output from "+commands+":\n"+new String(stdout.toByteArray())); + return new String(stdout.toByteArray()); + } +} http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/31b5c079/core/src/test/java/brooklyn/location/basic/jclouds/StandaloneJcloudsTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/location/basic/jclouds/StandaloneJcloudsTest.java b/core/src/test/java/brooklyn/location/basic/jclouds/StandaloneJcloudsTest.java index af199e8..724f282 100644 --- a/core/src/test/java/brooklyn/location/basic/jclouds/StandaloneJcloudsTest.java +++ b/core/src/test/java/brooklyn/location/basic/jclouds/StandaloneJcloudsTest.java @@ -1,35 +1,41 @@ package brooklyn.location.basic.jclouds; +import java.io.File; +import java.nio.charset.Charset; +import java.util.Arrays; import java.util.Properties; import java.util.Set; import java.util.UUID; import org.jclouds.Constants; -import org.jclouds.aws.ec2.reference.AWSEC2Constants; +import org.jclouds.ContextBuilder; import org.jclouds.compute.ComputeService; -import org.jclouds.compute.ComputeServiceContextFactory; +import org.jclouds.compute.ComputeServiceContext; import org.jclouds.compute.RunNodesException; import org.jclouds.compute.domain.ExecResponse; import org.jclouds.compute.domain.NodeMetadata; import org.jclouds.compute.domain.Template; import org.jclouds.compute.domain.TemplateBuilder; import org.jclouds.compute.options.RunScriptOptions; +import org.jclouds.compute.options.TemplateOptions; import org.jclouds.domain.Credentials; import org.jclouds.domain.LoginCredentials; import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; import org.jclouds.scriptbuilder.domain.Statement; import org.jclouds.scriptbuilder.domain.Statements; +import org.jclouds.scriptbuilder.statements.login.AdminAccess; import org.jclouds.sshj.config.SshjSshClientModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.Assert; import org.testng.annotations.Test; import brooklyn.config.BrooklynProperties; +import brooklyn.util.text.Identifiers; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import com.google.inject.Module; +import com.google.common.io.Files; public class StandaloneJcloudsTest { @@ -40,30 +46,27 @@ public class StandaloneJcloudsTest { String identity = globals.getFirst("brooklyn.jclouds.aws-ec2.identity"); String credential = globals.getFirst("brooklyn.jclouds.aws-ec2.credential"); - @Test(groups={"WIP","Integration"}) + @Test(groups={"WIP","Live"}) public void createVm() { String groupId = "mygroup-"+System.getProperty("user.name")+"-"+UUID.randomUUID().toString(); Properties properties = new Properties(); - properties.setProperty(Constants.PROPERTY_PROVIDER, "aws-ec2"); - properties.setProperty(Constants.PROPERTY_IDENTITY, identity); - properties.setProperty(Constants.PROPERTY_CREDENTIAL, credential); properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true)); properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true)); - - properties.setProperty(AWSEC2Constants.PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine"); + // handy to list all images... but very slow! +// properties.setProperty(AWSEC2Constants.PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine"); - Iterable<Module> modules = ImmutableSet.<Module> of(new SshjSshClientModule(), new SLF4JLoggingModule()); - - ComputeServiceContextFactory computeServiceFactory = new ComputeServiceContextFactory(); + ComputeServiceContext computeServiceContext = ContextBuilder.newBuilder("aws-ec2"). + modules(Arrays.asList(new SshjSshClientModule(), new SLF4JLoggingModule())). + credentials(identity, credential). + overrides(properties). + build(ComputeServiceContext.class); - final ComputeService computeService = computeServiceFactory - .createContext("aws-ec2", modules, properties) - .getComputeService(); + final ComputeService computeService = computeServiceContext.getComputeService(); NodeMetadata node = null; try { - LOG.info("Creating VM"); + LOG.info("Creating VM for "+identity); TemplateBuilder templateBuilder = computeService.templateBuilder(); templateBuilder.locationId("eu-west-1"); @@ -121,4 +124,102 @@ public class StandaloneJcloudsTest { } } + + @Test(groups={"WIP","Live"}) + public void createVmWithAdminUser() { + String groupId = "mygroup-"+System.getProperty("user.name")+"-"+UUID.randomUUID().toString(); + + Properties properties = new Properties(); + properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true)); + properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true)); + + ComputeServiceContext computeServiceContext = ContextBuilder.newBuilder("aws-ec2"). + modules(Arrays.asList(new SshjSshClientModule(), new SLF4JLoggingModule())). + credentials(identity, credential). + overrides(properties). + build(ComputeServiceContext.class); + + final ComputeService computeService = computeServiceContext.getComputeService(); + + NodeMetadata node = null; + try { + LOG.info("Creating VM for "+identity); + String myPubKey = Files.toString(new File(System.getProperty("user.home")+"/.ssh/aws-id_rsa.pub"), Charset.defaultCharset()); + String myPrivKey = Files.toString(new File(System.getProperty("user.home")+"/.ssh/aws-id_rsa"), Charset.defaultCharset()); + + TemplateBuilder templateBuilder = computeService.templateBuilder(); + templateBuilder.locationId("us-east-1"); + TemplateOptions opts = new TemplateOptions(); + +// templateBuilder.imageId("us-east-1/ami-2342a94a"); //rightscale + // either use above, or below + templateBuilder.imageId("us-east-1/ami-f95cf390"); //private one (to test when user isn't autodetected) + opts.overrideLoginUser("ec2-user"); + + AdminAccess.Builder adminBuilder = AdminAccess.builder(). + adminUsername("bob"). + grantSudoToAdminUser(true). + authorizeAdminPublicKey(true).adminPublicKey(myPubKey). + // items below aren't wanted but values for some are required otherwise AdminAccess uses all defaults + lockSsh(true).adminPassword(Identifiers.makeRandomId(12)). + resetLoginPassword(false).loginPassword(Identifiers.makeRandomId(12)). + installAdminPrivateKey(false).adminPrivateKey("ignored"); + opts.runScript(adminBuilder.build()); + + templateBuilder.options(opts); + + Template template = templateBuilder.build(); + Set<? extends NodeMetadata> nodes = computeService.createNodesInGroup(groupId, 1, template); + node = Iterables.getOnlyElement(nodes, null); + if (node == null) throw new IllegalStateException("No nodes returned"); + + LOG.info("Started VM, waiting for it to be sshable on "+node.getPublicAddresses()); + final LoginCredentials crds = +// node.getCredentials(); + LoginCredentials.builder().user("bob").privateKey(myPrivKey).build(); + boolean reachable = false; + for (int i=0; i<120; i++) { + try { + Statement statement = Statements.newStatementList(Statements.exec("date")); + ExecResponse response = computeService.runScriptOnNode(node.getId(), statement, + RunScriptOptions.Builder.overrideLoginCredentials(crds)); + if (response.getExitStatus() == 0) { + LOG.info("ssh 'date' succeeded"); + reachable = true; + break; + } + LOG.info("ssh 'date' failed, exit "+response.getExitStatus()+", but still in retry loop"); + } catch (Exception e) { + if (i<120) + LOG.info("ssh 'date' failed, but still in retry loop: "+e); + else { + LOG.error("ssh 'date' failed after timeout: "+e, e); + Throwables.propagate(e); + } + } + Thread.sleep(1000); + } + + if (!reachable) { + throw new IllegalStateException("SSH failed, never reachable"); + } + + } catch (RunNodesException e) { + if (e.getNodeErrors().size() > 0) { + node = Iterables.get(e.getNodeErrors().keySet(), 0); + } + LOG.error("Failed to start VM: "+e, e); + throw Throwables.propagate(e); + } catch (Exception e) { + LOG.error("Failed to start VM: "+e, e); + throw Throwables.propagate(e); + } finally { + LOG.info("Now destroying VM: "+node); + computeService.destroyNode( node.getId() ); + + computeService.getContext().close(); + } + + } + }
