spark git commit: updated ec2 instance types
Repository: spark
Updated Branches:
refs/heads/master 35c9599b9 -> 1c78f6866
updated ec2 instance types
I needed to run some d2 instances, so I updated the spark_ec2.py accordingly
Author: Brendan Collins
Closes #6014 from brendancol/ec2-instance-types-update and squashes the
following commits:
d7b4191 [Brendan Collins] Merge branch 'ec2-instance-types-update' of
github.com:brendancol/spark into ec2-instance-types-update
6366c45 [Brendan Collins] added back cc1.4xlarge
fc2931f [Brendan Collins] updated ec2 instance types
80c2aa6 [Brendan Collins] vertically aligned whitespace
85c6236 [Brendan Collins] vertically aligned whitespace
1657c26 [Brendan Collins] updated ec2 instance types
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1c78f686
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1c78f686
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1c78f686
Branch: refs/heads/master
Commit: 1c78f6866ebbcfb41d9875bfa3c0b9fa23b188bf
Parents: 35c9599
Author: Brendan Collins
Authored: Fri May 8 15:59:34 2015 -0700
Committer: Shivaram Venkataraman
Committed: Fri May 8 15:59:34 2015 -0700
--
ec2/spark_ec2.py | 70 ++-
1 file changed, 47 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1c78f686/ec2/spark_ec2.py
--
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py
index 87c0818..ab4a96f 100755
--- a/ec2/spark_ec2.py
+++ b/ec2/spark_ec2.py
@@ -347,46 +347,57 @@ def get_validate_spark_version(version, repo):
# Source: http://aws.amazon.com/amazon-linux-ami/instance-type-matrix/
-# Last Updated: 2014-06-20
+# Last Updated: 2015-05-08
# For easy maintainability, please keep this manually-inputted dictionary
sorted by key.
EC2_INSTANCE_TYPES = {
"c1.medium": "pvm",
"c1.xlarge": "pvm",
+"c3.large":"pvm",
+"c3.xlarge": "pvm",
"c3.2xlarge": "pvm",
"c3.4xlarge": "pvm",
"c3.8xlarge": "pvm",
-"c3.large":"pvm",
-"c3.xlarge": "pvm",
+"c4.large":"hvm",
+"c4.xlarge": "hvm",
+"c4.2xlarge": "hvm",
+"c4.4xlarge": "hvm",
+"c4.8xlarge": "hvm",
"cc1.4xlarge": "hvm",
"cc2.8xlarge": "hvm",
"cg1.4xlarge": "hvm",
"cr1.8xlarge": "hvm",
+"d2.xlarge": "hvm",
+"d2.2xlarge": "hvm",
+"d2.4xlarge": "hvm",
+"d2.8xlarge": "hvm",
+"g2.2xlarge": "hvm",
+"g2.8xlarge": "hvm",
"hi1.4xlarge": "pvm",
"hs1.8xlarge": "pvm",
+"i2.xlarge": "hvm",
"i2.2xlarge": "hvm",
"i2.4xlarge": "hvm",
"i2.8xlarge": "hvm",
-"i2.xlarge": "hvm",
-"m1.large":"pvm",
-"m1.medium": "pvm",
"m1.small":"pvm",
+"m1.medium": "pvm",
+"m1.large":"pvm",
"m1.xlarge": "pvm",
+"m2.xlarge": "pvm",
"m2.2xlarge": "pvm",
"m2.4xlarge": "pvm",
-"m2.xlarge": "pvm",
-"m3.2xlarge": "hvm",
-"m3.large":"hvm",
"m3.medium": "hvm",
+"m3.large":"hvm",
"m3.xlarge": "hvm",
+"m3.2xlarge": "hvm",
+"r3.large":"hvm",
+"r3.xlarge": "hvm",
"r3.2xlarge": "hvm",
"r3.4xlarge": "hvm",
"r3.8xlarge": "hvm",
-"r3.large":"hvm",
-"r3.xlarge": "hvm",
"t1.micro":"pvm",
-"t2.medium": "hvm",
"t2.micro":"hvm",
"t2.small":"hvm",
+"t2.medium": "hvm",
}
@@ -878,44 +889,57 @@ def wait_for_cluster_state(conn, opts, cluster_instances,
cluster_state):
# Get number of local disks available for a given EC2 instance type.
def get_num_disks(instance_type):
# Source:
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html
-# Last Updated: 2014-06-20
+# Last Updated: 2015-05-08
# For easy maintainability, please keep this manually-inputted dictionary
sorted by key.
disks_by_instance = {
"c1.medium": 1,
"c1.xlarge": 4,
+"c3.large":2,
+"c3.xlarge": 2,
"c3.2xlarge": 2,
"c3.4xlarge": 2,
"c3.8xlarge": 2,
-"c3.large":2,
-"c3.xlarge": 2,
+"c4.large":0,
+"c4.xlarge": 0,
+"c4.2xlarge": 0,
+"c4.4xlarge": 0,
+"c4.8xlarge": 0,
"cc1.4xlarge": 2,
"cc2.8xlarge": 4,
"cg1.4xlarge": 2,
"cr1.8xlarge": 2,
+"d2.xlarge": 3,
+"d2.2xlarge": 6,
+"d2.4xlarge": 12,
+"d2.8xlarge": 24,
"g2.2xlarge": 1,
+"g2.8xlarge": 2,
"hi1.4xlarge": 2,
"hs1.8xlarge": 24,
+"i2.xlarge": 1,
"i2.2xlarge": 2,
"i2.4xlarge": 4,
"i2.8xlarge": 8,
-"i2.xlarge": 1,
-"m1.large":2,
-
spark git commit: updated ec2 instance types
Repository: spark
Updated Branches:
refs/heads/branch-1.4 ab48df391 -> 6e35cb555
updated ec2 instance types
I needed to run some d2 instances, so I updated the spark_ec2.py accordingly
Author: Brendan Collins
Closes #6014 from brendancol/ec2-instance-types-update and squashes the
following commits:
d7b4191 [Brendan Collins] Merge branch 'ec2-instance-types-update' of
github.com:brendancol/spark into ec2-instance-types-update
6366c45 [Brendan Collins] added back cc1.4xlarge
fc2931f [Brendan Collins] updated ec2 instance types
80c2aa6 [Brendan Collins] vertically aligned whitespace
85c6236 [Brendan Collins] vertically aligned whitespace
1657c26 [Brendan Collins] updated ec2 instance types
(cherry picked from commit 1c78f6866ebbcfb41d9875bfa3c0b9fa23b188bf)
Signed-off-by: Shivaram Venkataraman
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6e35cb55
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6e35cb55
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6e35cb55
Branch: refs/heads/branch-1.4
Commit: 6e35cb555ae668f6d47b806658404af9497acf6c
Parents: ab48df3
Author: Brendan Collins
Authored: Fri May 8 15:59:34 2015 -0700
Committer: Shivaram Venkataraman
Committed: Fri May 8 15:59:54 2015 -0700
--
ec2/spark_ec2.py | 70 ++-
1 file changed, 47 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6e35cb55/ec2/spark_ec2.py
--
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py
index 87c0818..ab4a96f 100755
--- a/ec2/spark_ec2.py
+++ b/ec2/spark_ec2.py
@@ -347,46 +347,57 @@ def get_validate_spark_version(version, repo):
# Source: http://aws.amazon.com/amazon-linux-ami/instance-type-matrix/
-# Last Updated: 2014-06-20
+# Last Updated: 2015-05-08
# For easy maintainability, please keep this manually-inputted dictionary
sorted by key.
EC2_INSTANCE_TYPES = {
"c1.medium": "pvm",
"c1.xlarge": "pvm",
+"c3.large":"pvm",
+"c3.xlarge": "pvm",
"c3.2xlarge": "pvm",
"c3.4xlarge": "pvm",
"c3.8xlarge": "pvm",
-"c3.large":"pvm",
-"c3.xlarge": "pvm",
+"c4.large":"hvm",
+"c4.xlarge": "hvm",
+"c4.2xlarge": "hvm",
+"c4.4xlarge": "hvm",
+"c4.8xlarge": "hvm",
"cc1.4xlarge": "hvm",
"cc2.8xlarge": "hvm",
"cg1.4xlarge": "hvm",
"cr1.8xlarge": "hvm",
+"d2.xlarge": "hvm",
+"d2.2xlarge": "hvm",
+"d2.4xlarge": "hvm",
+"d2.8xlarge": "hvm",
+"g2.2xlarge": "hvm",
+"g2.8xlarge": "hvm",
"hi1.4xlarge": "pvm",
"hs1.8xlarge": "pvm",
+"i2.xlarge": "hvm",
"i2.2xlarge": "hvm",
"i2.4xlarge": "hvm",
"i2.8xlarge": "hvm",
-"i2.xlarge": "hvm",
-"m1.large":"pvm",
-"m1.medium": "pvm",
"m1.small":"pvm",
+"m1.medium": "pvm",
+"m1.large":"pvm",
"m1.xlarge": "pvm",
+"m2.xlarge": "pvm",
"m2.2xlarge": "pvm",
"m2.4xlarge": "pvm",
-"m2.xlarge": "pvm",
-"m3.2xlarge": "hvm",
-"m3.large":"hvm",
"m3.medium": "hvm",
+"m3.large":"hvm",
"m3.xlarge": "hvm",
+"m3.2xlarge": "hvm",
+"r3.large":"hvm",
+"r3.xlarge": "hvm",
"r3.2xlarge": "hvm",
"r3.4xlarge": "hvm",
"r3.8xlarge": "hvm",
-"r3.large":"hvm",
-"r3.xlarge": "hvm",
"t1.micro":"pvm",
-"t2.medium": "hvm",
"t2.micro":"hvm",
"t2.small":"hvm",
+"t2.medium": "hvm",
}
@@ -878,44 +889,57 @@ def wait_for_cluster_state(conn, opts, cluster_instances,
cluster_state):
# Get number of local disks available for a given EC2 instance type.
def get_num_disks(instance_type):
# Source:
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html
-# Last Updated: 2014-06-20
+# Last Updated: 2015-05-08
# For easy maintainability, please keep this manually-inputted dictionary
sorted by key.
disks_by_instance = {
"c1.medium": 1,
"c1.xlarge": 4,
+"c3.large":2,
+"c3.xlarge": 2,
"c3.2xlarge": 2,
"c3.4xlarge": 2,
"c3.8xlarge": 2,
-"c3.large":2,
-"c3.xlarge": 2,
+"c4.large":0,
+"c4.xlarge": 0,
+"c4.2xlarge": 0,
+"c4.4xlarge": 0,
+"c4.8xlarge": 0,
"cc1.4xlarge": 2,
"cc2.8xlarge": 4,
"cg1.4xlarge": 2,
"cr1.8xlarge": 2,
+"d2.xlarge": 3,
+"d2.2xlarge": 6,
+"d2.4xlarge": 12,
+"d2.8xlarge": 24,
"g2.2xlarge": 1,
+"g2.8xlarge": 2,
"hi1.4xlarge": 2,
"hs1.8xlarge": 24,
+"i2.xlarge": 1,
"i2.2xlarge
