This is an automated email from the ASF dual-hosted git repository.
jbarrett pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git
The following commit(s) were added to refs/heads/develop by this push:
new 6cb68ee Create dedicated hosts (#109)
6cb68ee is described below
commit 6cb68ee7b78b0b439c7257f0d568485f04c75101
Author: Jacob Barrett <jbarr...@pivotal.io>
AuthorDate: Tue Sep 17 19:52:50 2019 -0700
Create dedicated hosts (#109)
---
infrastructure/scripts/aws/run_tests.sh | 4 +-
.../geode/infrastructure/aws/DestroyCluster.java | 20 +++++++
.../geode/infrastructure/aws/LaunchCluster.java | 61 ++++++++++++++--------
3 files changed, 61 insertions(+), 24 deletions(-)
diff --git a/infrastructure/scripts/aws/run_tests.sh
b/infrastructure/scripts/aws/run_tests.sh
index 3dd4d0d..9a77404 100755
--- a/infrastructure/scripts/aws/run_tests.sh
+++ b/infrastructure/scripts/aws/run_tests.sh
@@ -17,7 +17,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e -o pipefail
+set -x -e -o pipefail
DEFAULT_BENCHMARK_REPO='https://github.com/apache/geode-benchmarks'
BENCHMARK_REPO=${DEFAULT_BENCHMARK_REPO}
@@ -144,7 +144,7 @@ REPO=$(fixRepoName ${REPO})
SSH_OPTIONS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i
~/.geode-benchmarks/${TAG}-privkey.pem"
HOSTS=`aws ec2 describe-instances --query
'Reservations[*].Instances[*].PrivateIpAddress' --filter
"Name=tag:geode-benchmarks,Values=${TAG}" --output text`
HOSTS=$(echo ${HOSTS} | tr ' ' ',')
-FIRST_INSTANCE=`aws ec2 describe-instances --query
'Reservations[*].Instances[*].PublicIpAddress' --filter
"Name=tag:geode-benchmarks,Values=${TAG}" --output text | cut -f 1`
+FIRST_INSTANCE=`aws ec2 describe-instances --query
'Reservations[*].Instances[*].PublicIpAddress' --filter
"Name=tag:geode-benchmarks,Values=${TAG}" --output text | cut -f 1 | head -n 1`
echo "FIRST_INSTANCE=${FIRST_INSTANCE}"
echo "HOSTS=${HOSTS}"
diff --git
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/DestroyCluster.java
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/DestroyCluster.java
index 1c7b668..7156168 100644
---
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/DestroyCluster.java
+++
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/DestroyCluster.java
@@ -29,10 +29,14 @@ import
software.amazon.awssdk.services.ec2.model.DeleteKeyPairRequest;
import software.amazon.awssdk.services.ec2.model.DeleteLaunchTemplateRequest;
import software.amazon.awssdk.services.ec2.model.DeletePlacementGroupRequest;
import software.amazon.awssdk.services.ec2.model.DeleteSecurityGroupRequest;
+import software.amazon.awssdk.services.ec2.model.DescribeHostsRequest;
+import software.amazon.awssdk.services.ec2.model.DescribeHostsResponse;
import software.amazon.awssdk.services.ec2.model.DescribeInstancesRequest;
import software.amazon.awssdk.services.ec2.model.DescribeInstancesResponse;
import software.amazon.awssdk.services.ec2.model.Filter;
+import software.amazon.awssdk.services.ec2.model.Host;
import software.amazon.awssdk.services.ec2.model.Instance;
+import software.amazon.awssdk.services.ec2.model.ReleaseHostsRequest;
import software.amazon.awssdk.services.ec2.model.TerminateInstancesRequest;
import org.apache.geode.infrastructure.BenchmarkMetadata;
@@ -51,6 +55,7 @@ public class DestroyCluster {
}
deleteInstances(benchmarkTag);
+ releaseHosts(benchmarkTag);
deleteLaunchTemplate(benchmarkTag);
deleteSecurityGroup(benchmarkTag);
deletePlacementGroup(benchmarkTag);
@@ -58,6 +63,21 @@ public class DestroyCluster {
deleteMetadata(benchmarkTag);
}
+ private static void releaseHosts(String benchmarkTag) {
+ DescribeHostsResponse hosts =
ec2.describeHosts(DescribeHostsRequest.builder()
+ .filter(Filter.builder()
+ .name("tag:" + BenchmarkMetadata.PREFIX)
+ .values(benchmarkTag)
+ .build())
+ .build());
+
+ List<String> hostIds =
hosts.hosts().stream().map(Host::hostId).collect(Collectors.toList());
+
+ ec2.releaseHosts(ReleaseHostsRequest.builder().hostIds(hostIds).build());
+
+ System.out.println("Hosts for cluster '" + benchmarkTag + "' released.");
+ }
+
private static void deleteKeyPair(String benchmarkTag) {
try {
System.out.println("Deleting cluster keypair: " +
AwsBenchmarkMetadata.keyPair(benchmarkTag));
diff --git
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
index 339fbd0..879178d 100644
---
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
+++
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
@@ -35,6 +35,8 @@ import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONObject;
import software.amazon.awssdk.services.ec2.Ec2Client;
+import software.amazon.awssdk.services.ec2.model.AllocateHostsRequest;
+import software.amazon.awssdk.services.ec2.model.AllocateHostsResponse;
import
software.amazon.awssdk.services.ec2.model.AuthorizeSecurityGroupIngressRequest;
import software.amazon.awssdk.services.ec2.model.CreateKeyPairRequest;
import software.amazon.awssdk.services.ec2.model.CreateKeyPairResponse;
@@ -55,14 +57,15 @@ import software.amazon.awssdk.services.ec2.model.Image;
import software.amazon.awssdk.services.ec2.model.Instance;
import
software.amazon.awssdk.services.ec2.model.LaunchTemplateBlockDeviceMappingRequest;
import
software.amazon.awssdk.services.ec2.model.LaunchTemplateEbsBlockDeviceRequest;
-import
software.amazon.awssdk.services.ec2.model.LaunchTemplatePlacementRequest;
import software.amazon.awssdk.services.ec2.model.LaunchTemplateSpecification;
+import software.amazon.awssdk.services.ec2.model.Placement;
import software.amazon.awssdk.services.ec2.model.RequestLaunchTemplateData;
import software.amazon.awssdk.services.ec2.model.ResourceType;
import software.amazon.awssdk.services.ec2.model.RunInstancesRequest;
import software.amazon.awssdk.services.ec2.model.RunInstancesResponse;
import software.amazon.awssdk.services.ec2.model.Tag;
import software.amazon.awssdk.services.ec2.model.TagSpecification;
+import software.amazon.awssdk.services.ec2.model.Tenancy;
import software.amazon.awssdk.services.ec2.model.VolumeType;
import org.apache.geode.infrastructure.BenchmarkMetadata;
@@ -93,7 +96,8 @@ public class LaunchCluster {
authorizeSecurityGroup(benchmarkTag);
createLaunchTemplate(benchmarkTag, newestImage);
- List<String> instanceIds = launchInstances(benchmarkTag, tags, count);
+ List<String> hostIds = allocateHosts(tags, count);
+ List<String> instanceIds = launchInstances(benchmarkTag, tags, count,
hostIds);
DescribeInstancesResponse instances = waitForInstances(instanceIds);
List<String> publicIps = getPublicIps(instances);
createMetadata(benchmarkTag, publicIps);
@@ -113,27 +117,44 @@ public class LaunchCluster {
throw new IllegalStateException(s);
}
- private static List<String> launchInstances(String benchmarkTag, List<Tag>
tags,
- int instanceCount)
- throws InterruptedException {
- // launch instances
-
- RunInstancesResponse rir = ec2.runInstances(RunInstancesRequest.builder()
- .launchTemplate(LaunchTemplateSpecification.builder()
-
.launchTemplateName(AwsBenchmarkMetadata.launchTemplate(benchmarkTag))
- .build())
+ private static List<String> allocateHosts(List<Tag> tags, int count) {
+ AllocateHostsResponse hosts =
ec2.allocateHosts(AllocateHostsRequest.builder()
+ .availabilityZone("us-west-2a")
+ .instanceType(AwsBenchmarkMetadata.instanceType().toString())
+ .quantity(count)
.tagSpecifications(TagSpecification.builder()
.tags(tags)
- .resourceType(ResourceType.INSTANCE)
+ .resourceType(ResourceType.DEDICATED_HOST)
.build())
- .minCount(instanceCount)
- .maxCount(instanceCount)
.build());
- List<String> instanceIds = rir.instances()
- .stream()
- .map(Instance::instanceId)
- .collect(Collectors.toList());
+ return hosts.hostIds();
+ }
+
+ private static List<String> launchInstances(String launchTemplate, List<Tag>
tags,
+ int instanceCount, List<String> hosts)
+ throws InterruptedException {
+ List<String> instanceIds = new ArrayList<>(instanceCount);
+ for (String host : hosts) {
+ // launch instances
+ RunInstancesResponse rir = ec2.runInstances(RunInstancesRequest.builder()
+ .launchTemplate(LaunchTemplateSpecification.builder()
+
.launchTemplateName(AwsBenchmarkMetadata.launchTemplate(launchTemplate))
+ .build())
+ .placement(Placement.builder()
+ .tenancy(Tenancy.HOST)
+ .hostId(host)
+ .build())
+ .tagSpecifications(TagSpecification.builder()
+ .tags(tags)
+ .resourceType(ResourceType.INSTANCE)
+ .build())
+ .minCount(1)
+ .maxCount(1)
+ .build());
+
+ instanceIds.add(rir.instances().get(0).instanceId());
+ }
return instanceIds;
}
@@ -222,10 +243,6 @@ public class LaunchCluster {
.launchTemplateData(RequestLaunchTemplateData.builder()
.imageId(newestImage.imageId())
.instanceType(AwsBenchmarkMetadata.instanceType())
- .placement(LaunchTemplatePlacementRequest.builder()
-
.groupName(AwsBenchmarkMetadata.placementGroup(benchmarkTag))
- .tenancy(AwsBenchmarkMetadata.tenancy())
- .build())
.keyName(AwsBenchmarkMetadata.keyPair(benchmarkTag))
.securityGroups(securityGroupList)
.blockDeviceMappings(LaunchTemplateBlockDeviceMappingRequest.builder()
