HIVE-14849: Support google-compute-engine provider on Hive ptest framework (Sergio Pena, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/291f3d50 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/291f3d50 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/291f3d50 Branch: refs/heads/repl2 Commit: 291f3d503d5a8627f86ef5f7fdd7880d8da4760c Parents: cf72a73 Author: Sergio Pena <sergio.p...@cloudera.com> Authored: Wed Sep 28 21:33:00 2016 -0500 Committer: Sergio Pena <sergio.p...@cloudera.com> Committed: Wed Sep 28 21:33:00 2016 -0500 ---------------------------------------------------------------------- .../ptest2/conf/cloudhost.properties.example | 37 +++ testutils/ptest2/pom.xml | 5 + .../execution/context/CloudComputeService.java | 224 +++++++++++++++---- .../context/CloudExecutionContextProvider.java | 105 +++++++-- 4 files changed, 311 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/conf/cloudhost.properties.example ---------------------------------------------------------------------- diff --git a/testutils/ptest2/conf/cloudhost.properties.example b/testutils/ptest2/conf/cloudhost.properties.example new file mode 100644 index 0000000..c336052 --- /dev/null +++ b/testutils/ptest2/conf/cloudhost.properties.example @@ -0,0 +1,37 @@ +# +# This is just an example of different cloudhost providers +# + +# This context provides configurations for AWS EC2 and GCE (google compute engine) +executionContextProvider = org.apache.hive.ptest.execution.context.CloudExecutionContextProvider$Builder + +# Option: GCE +cloudProvider = google-compute-engine +gceJsonFile = # GCE JSON KEY FILE +instanceType = https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/zones/us-central1-a/machineTypes/n1-standard-8 +imageId = https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/global/images/hive-ptest-debian-8-20160927 +# keyPair = # UNUSED +securityGroup = hive-ptest + +# Option: AWS +cloudProvider = aws-ec2 +apiKey = # AWS ACCESS KEY +accessKey = # AWS SECRET ACCESS KEY +instanceType = c3.2xlarge +imageId = us-west-1/ami-1fa1445b +keyPair = hive-ptest +securityGroup = hive-ptest + +# Generic options +workingDirectory = /data/hive-ptest +profileDirectory = /usr/local/hiveptest/etc/public/ +privateKey = /home/hiveptest/.ssh/hive-ptest-user-key +dataDir = /data/hive-ptest/data/ +numHosts = 12 +groupName = hive-ptest-slaves +localDirs = /home/hiveptest/ +user = hiveptest +numThreads = 2 +maxLogDirectoriesPerProfile = 30 +userMetadata.owner = # USER +maxHostsPerCreateRequest = 12 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/pom.xml ---------------------------------------------------------------------- diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml index cea29b6..97981fb 100644 --- a/testutils/ptest2/pom.xml +++ b/testutils/ptest2/pom.xml @@ -107,6 +107,11 @@ limitations under the License. <version>${jclouds.version}</version> </dependency> <dependency> + <groupId>org.apache.jclouds.labs</groupId> + <artifactId>google-compute-engine</artifactId> + <version>${jclouds.version}</version> + </dependency> + <dependency> <groupId>org.apache.jclouds.driver</groupId> <artifactId>jclouds-sshj</artifactId> <version>${jclouds.version}</version> http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java index 64ee68e..e26c5ca 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java @@ -18,11 +18,13 @@ */ package org.apache.hive.ptest.execution.context; -import java.util.Collections; -import java.util.Properties; -import java.util.Map; -import java.util.Set; +import java.io.File; +import java.io.IOException; +import java.util.*; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; import org.jclouds.Constants; import org.jclouds.ContextBuilder; import org.jclouds.aws.ec2.compute.AWSEC2TemplateOptions; @@ -34,7 +36,12 @@ import org.jclouds.compute.domain.ComputeMetadata; import org.jclouds.compute.domain.NodeMetadata; import org.jclouds.compute.domain.NodeMetadata.Status; import org.jclouds.compute.domain.Template; +import org.jclouds.compute.options.TemplateOptions; +import org.jclouds.domain.Credentials; +import org.jclouds.googlecloud.GoogleCredentialsFromJson; +import org.jclouds.googlecomputeengine.compute.options.GoogleComputeEngineTemplateOptions; import org.jclouds.logging.log4j.config.Log4JLoggingModule; +import org.jclouds.sshj.config.SshjSshClientModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,55 +50,76 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import static com.google.common.base.Charsets.UTF_8; + public class CloudComputeService { private static final Logger LOG = LoggerFactory .getLogger(CloudComputeService.class); + private final CloudComputeConfig mConfig; private final ComputeServiceContext mComputeServiceContext; private final ComputeService mComputeService; - private final String mInstanceType; - private final String mGroupName; + private final Template mTemplate; private final String mGroupTag; - private final String mImageId; - private final String mkeyPair; - private final String mSecurityGroup; - private final Map<String, String> mUserMetadata; - /** - * JClouds requests on-demand instances when null - */ - private final Float mMaxBid; - public CloudComputeService(String apiKey, String accessKey, String instanceType, String groupName, - String imageId, String keyPair, String securityGroup, Float maxBid, Map<String,String> userMetadata) { - mInstanceType = instanceType; - mGroupName = groupName; - mImageId = imageId; - mkeyPair = keyPair; - mSecurityGroup = securityGroup; - mMaxBid = maxBid; - mGroupTag = "group=" + mGroupName; - mUserMetadata = userMetadata; + + public CloudComputeService(final CloudComputeConfig config) { + mConfig = config; + + mComputeServiceContext = initComputeServiceContext(config.getmProvider(), config.getIdentity(), config.getCredential()); + mComputeService = mComputeServiceContext.getComputeService(); + mTemplate = mComputeService.templateBuilder().hardwareId(config.getInstanceType()).imageId(config.getImageId()).build(); + + TemplateOptions options = mTemplate.getOptions(); + + // Set generic options + options.blockOnPort(22, 60); + options.userMetadata(config.getUserMetaData()); + + // Set provider options + switch (config.getmProvider()) { + case AWS: + mGroupTag = String.format("group=%s", config.getGroupName()); + + options.as(AWSEC2TemplateOptions.class) + .keyPair(config.getKeyPairName()) + .securityGroupIds(config.getSecurityGroup()) + .spotPrice(config.getMaxBid()) + .tags(Collections.singletonList(mGroupTag)); + break; + case GCE: + mGroupTag = config.getGroupName(); + + options.as(GoogleComputeEngineTemplateOptions.class) + .tags(Arrays.asList(config.getGroupName(), config.getSecurityGroup())); // GCE firewall is set through instance tags + break; + default: + mGroupTag = ""; + } + } + + private ComputeServiceContext initComputeServiceContext(CloudComputeConfig.CloudComputeProvider provider, String identity, String credential) { Properties overrides = new Properties(); + overrides.put(ComputeServiceProperties.POLL_INITIAL_PERIOD, String.valueOf(60L * 1000L)); overrides.put(ComputeServiceProperties.POLL_MAX_PERIOD, String.valueOf(600L * 1000L)); overrides.put(Constants.PROPERTY_MAX_RETRIES, String.valueOf(60)); - mComputeServiceContext = ContextBuilder.newBuilder("aws-ec2") - .credentials(apiKey, accessKey) - .modules(ImmutableSet.of(new Log4JLoggingModule())) - .overrides(overrides) - .buildView(ComputeServiceContext.class); - mComputeService = mComputeServiceContext.getComputeService(); + + return ContextBuilder.newBuilder(provider.getmJcloudsId()) + .credentials(identity, credential) + .modules(ImmutableSet.of( + new SshjSshClientModule(), + new Log4JLoggingModule() + )) + .overrides(overrides) + .buildView(ComputeServiceContext.class); } + public Set<NodeMetadata> createNodes(int count) throws RunNodesException { Set<NodeMetadata> result = Sets.newHashSet(); - Template template = mComputeService.templateBuilder() - .hardwareId(mInstanceType).imageId(mImageId).build(); - template.getOptions().as(AWSEC2TemplateOptions.class).keyPair(mkeyPair) - .securityGroupIds(mSecurityGroup).blockOnPort(22, 60) - .spotPrice(mMaxBid).tags(Collections.singletonList(mGroupTag)) - .userMetadata(mUserMetadata); - result.addAll(mComputeService.createNodesInGroup(mGroupName, count, template)); + result.addAll(mComputeService.createNodesInGroup(mConfig.getGroupName(), count, mTemplate)); return result; } + static Predicate<ComputeMetadata> createFilterPTestPredicate(final String groupName, final String groupTag) { return new Predicate<ComputeMetadata>() { @@ -122,7 +150,7 @@ public class CloudComputeService { public Set<NodeMetadata> listRunningNodes(){ Set<NodeMetadata> result = Sets.newHashSet(); result.addAll(mComputeService.listNodesDetailsMatching( - createFilterPTestPredicate(mGroupName, mGroupTag))); + createFilterPTestPredicate(mConfig.getGroupName(), mGroupTag))); return result; } public void destroyNode(String nodeId) { @@ -131,4 +159,124 @@ public class CloudComputeService { public void close() { mComputeServiceContext.close(); } + + public static class CloudComputeConfig { + public enum CloudComputeProvider { + AWS("aws-ec2"), + GCE("google-compute-engine"); + + private final String mJcloudsId; + + CloudComputeProvider(String jcloudsId) { + mJcloudsId = jcloudsId; + } + + public String getmJcloudsId() { + return mJcloudsId; + } + }; + + private final CloudComputeProvider mProvider; + + private String mIdentity; + private String mCredential; + private String mInstanceType; + private String mImageId; + private String mGroupName; + private String mSecurityGroup; + private String mKeyPairName; + private Map<String, String> mUserMetaData; + + /** + * JClouds requests on-demand instances when null + */ + private Float mMaxBid; + + public CloudComputeConfig(CloudComputeProvider provider) { + mProvider = provider; + } + + public void setCredentials(String identity, String credential) { + mIdentity = identity; + mCredential = credential; + } + + public void setInstanceType(String instanceType) { + mInstanceType = instanceType; + } + + public void setImageId(String imageId) { + mImageId = imageId; + } + + public void setGroupName(String groupName) { + mGroupName = groupName; + } + + public void setSecurityGroup(String securityGroup) { + mSecurityGroup = securityGroup; + } + + public void setMaxBid(Float maxBid) { + mMaxBid = maxBid; + } + + public void setmKeyPairName(String keyPairName) { + mKeyPairName = keyPairName; + } + + public void setUserMetaData(Map<String, String> userMetaData) { + mUserMetaData = userMetaData; + } + + public CloudComputeProvider getmProvider() { + return mProvider; + } + + public String getIdentity() { + return mIdentity; + } + + public String getCredential() { + return mCredential; + } + + public String getInstanceType() { + return mInstanceType; + } + + public String getImageId() { + return mImageId; + } + + public String getGroupName() { + return mGroupName; + } + + public String getSecurityGroup() { + return mSecurityGroup; + } + + public Float getMaxBid() { + return mMaxBid; + } + + public String getKeyPairName() { + return mKeyPairName; + } + + public Map<String, String> getUserMetaData() { + if (mUserMetaData == null) { + return ImmutableMap.of(); + } + + return mUserMetaData; + } + } + + public static Credentials getCredentialsFromJsonKeyFile(String filename) throws IOException { + String fileContents = Files.toString(new File(filename), UTF_8); + Supplier<Credentials> credentialSupplier = new GoogleCredentialsFromJson(fileContents); + return credentialSupplier.get(); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java index 343efde..8b82497 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java @@ -34,12 +34,14 @@ import java.util.concurrent.TimeUnit; import org.apache.hive.ptest.execution.Constants; import org.apache.hive.ptest.execution.Dirs; +import org.apache.hive.ptest.execution.LocalCommandFactory; import org.apache.hive.ptest.execution.conf.Context; import org.apache.hive.ptest.execution.conf.Host; import org.apache.hive.ptest.execution.ssh.SSHCommand; import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor; import org.jclouds.compute.RunNodesException; import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.domain.Credentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,16 +57,24 @@ import com.google.common.collect.Sets; public class CloudExecutionContextProvider implements ExecutionContextProvider { private static final Logger LOG = LoggerFactory .getLogger(CloudExecutionContextProvider.class); - public static final String DATA_DIR = "dataDir"; + public static final String CLOUD_PROVIDER = "cloudProvider"; + + // GCE settings + public static final String GCE_JSON_CREDS_FILE = "gceJsonFile"; + + // AWS settings public static final String API_KEY = "apiKey"; public static final String ACCESS_KEY = "accessKey"; + public static final String KEY_PAIR = "keyPair"; + public static final String MAX_BID = "maxBid"; + + // Generic settings + public static final String DATA_DIR = "dataDir"; public static final String NUM_HOSTS = "numHosts"; public static final String MAX_HOSTS_PER_CREATE_REQUEST = "maxHostsPerCreateRequest"; public static final String GROUP_NAME = "groupName"; public static final String IMAGE_ID = "imageId"; - public static final String KEY_PAIR = "keyPair"; public static final String SECURITY_GROUP = "securityGroup"; - public static final String MAX_BID = "maxBid"; public static final String SLAVE_LOCAL_DIRECTORIES = "localDirs"; public static final String USERNAME = "user"; public static final String INSTANCE_TYPE = "instanceType"; @@ -304,6 +314,7 @@ public class CloudExecutionContextProvider implements ExecutionContextProvider { LOG.error("Verify command still executing on a host after 10 minutes"); } } catch (InterruptedException e) { + terminateInternal(result); throw new CreateHostsFailedException("Interrupted while trying to create hosts", e); } finally { if(!executorService.isShutdown()) { @@ -418,29 +429,79 @@ public class CloudExecutionContextProvider implements ExecutionContextProvider { return create(context, workingDirectory); } } + + private static CloudComputeService createAwsService(final Context context) { + String apiKey = Preconditions.checkNotNull(context.getString(API_KEY), API_KEY + " is required"); + String accessKey = Preconditions.checkNotNull(context.getString(ACCESS_KEY), ACCESS_KEY + " is required"); + String imageId = Preconditions.checkNotNull(context.getString(IMAGE_ID), IMAGE_ID + " is required"); + String keyPair = Preconditions.checkNotNull(context.getString(KEY_PAIR), KEY_PAIR + " is required"); + String securityGroup = Preconditions.checkNotNull(context.getString(SECURITY_GROUP), SECURITY_GROUP + " is required"); + + Float maxBid = context.getFloat(MAX_BID); + Preconditions.checkArgument(maxBid == null || maxBid > 0, MAX_BID + " must be null or greater than zero"); + + String instanceType = context.getString(INSTANCE_TYPE, "c1.xlarge"); + String groupName = context.getString(GROUP_NAME, "hive-ptest-slaves"); + + CloudComputeService.CloudComputeConfig config = + new CloudComputeService.CloudComputeConfig(CloudComputeService.CloudComputeConfig.CloudComputeProvider.AWS); + + config.setCredentials(apiKey, accessKey); + config.setInstanceType(instanceType); + config.setGroupName(groupName); + config.setImageId(imageId); + config.setmKeyPairName(keyPair); + config.setSecurityGroup(securityGroup); + config.setMaxBid(maxBid); + config.setUserMetaData(context.getSubProperties(USER_METADATA + ".")); + + return new CloudComputeService(config); + } + + private static CloudComputeService createGceService(final Context context) throws IOException { + String gceJsonFile = Preconditions.checkNotNull(context.getString(GCE_JSON_CREDS_FILE), GCE_JSON_CREDS_FILE + " is required"); + String imageId = Preconditions.checkNotNull(context.getString(IMAGE_ID), IMAGE_ID + " is required"); + String securityGroup = Preconditions.checkNotNull(context.getString(SECURITY_GROUP), SECURITY_GROUP + " is required"); + String instanceType = Preconditions.checkNotNull(context.getString(INSTANCE_TYPE, ""), INSTANCE_TYPE + " is required"); + + String groupName = context.getString(GROUP_NAME, "hive-ptest-slaves"); + + CloudComputeService.CloudComputeConfig config = + new CloudComputeService.CloudComputeConfig(CloudComputeService.CloudComputeConfig.CloudComputeProvider.GCE); + + Credentials creds = + CloudComputeService.getCredentialsFromJsonKeyFile(gceJsonFile); + + config.setCredentials(creds.identity, creds.credential); + config.setInstanceType(instanceType); + config.setGroupName(groupName); + config.setImageId(imageId); + config.setSecurityGroup(securityGroup); + config.setUserMetaData(context.getSubProperties(USER_METADATA + ".")); + + return new CloudComputeService(config); + } + + private static CloudComputeService createService(final Context context) throws IOException { + String cloudProvider = context.getString(CLOUD_PROVIDER, "aws-ec2"); + + if (cloudProvider.equalsIgnoreCase("aws-ec2")) { + return createAwsService(context); + } else if (cloudProvider.equalsIgnoreCase("google-compute-engine")) { + return createGceService(context); + } else { + throw new IllegalArgumentException("Unknown cloud provider name: " + cloudProvider); + } + } private static CloudExecutionContextProvider create(Context context, String workingDirectory) throws IOException { String dataDir = Preconditions.checkNotNull(context.getString(DATA_DIR), DATA_DIR + " is required"); - String apiKey = Preconditions.checkNotNull(context.getString(API_KEY), - API_KEY + " is required"); - String accessKey = Preconditions.checkNotNull( - context.getString(ACCESS_KEY), ACCESS_KEY + " is required"); int maxHostsPerCreateRequest = context.getInteger(MAX_HOSTS_PER_CREATE_REQUEST, 2); Integer numHosts = context.getInteger(NUM_HOSTS, 8); Preconditions.checkArgument(numHosts > 0, NUM_HOSTS + " must be greater than zero"); - String groupName = context.getString(GROUP_NAME, "hive-ptest-slaves"); - String imageId = Preconditions.checkNotNull(context.getString(IMAGE_ID), - IMAGE_ID + " is required"); - String keyPair = Preconditions.checkNotNull(context.getString(KEY_PAIR), - KEY_PAIR + " is required"); - String securityGroup = Preconditions.checkNotNull( - context.getString(SECURITY_GROUP), SECURITY_GROUP + " is required"); - Float maxBid = context.getFloat(MAX_BID); - Preconditions.checkArgument(maxBid == null || maxBid > 0, MAX_BID - + " must be null or greater than zero"); String privateKey = Preconditions.checkNotNull( context.getString(PRIVATE_KEY), PRIVATE_KEY + " is required"); String user = context.getString(USERNAME, "hiveptest"); @@ -448,12 +509,12 @@ public class CloudExecutionContextProvider implements ExecutionContextProvider { .split(context.getString(SLAVE_LOCAL_DIRECTORIES, "/home/hiveptest/")), String.class); Integer numThreads = context.getInteger(NUM_THREADS, 3); - String instanceType = context.getString(INSTANCE_TYPE, "c1.xlarge"); - CloudComputeService cloudComputeService = new CloudComputeService(apiKey, accessKey, - instanceType, groupName, imageId, keyPair, securityGroup, maxBid, context.getSubProperties(USER_METADATA + ".")); + + CloudComputeService cloudComputeService = createService(context); CloudExecutionContextProvider service = new CloudExecutionContextProvider( - dataDir, numHosts, cloudComputeService, new SSHCommandExecutor(LOG), workingDirectory, - privateKey, user, localDirs, numThreads, 60, maxHostsPerCreateRequest); + dataDir, numHosts, cloudComputeService, + new SSHCommandExecutor(LOG, new LocalCommandFactory(LOG), "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"), + workingDirectory, privateKey, user, localDirs, numThreads, 60, maxHostsPerCreateRequest); return service; } }
