Yingyi Bu has submitted this change and it was merged. Change subject: Make aws scripts work with all latest instance types that are equiped with instance-stores. ......................................................................
Make aws scripts work with all latest instance types that are equiped with instance-stores. Change-Id: I16efff22a029e316c7a7dc0402c0131e0c0bdea4 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1603 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> BAD: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Ian Maxon <ima...@apache.org> --- M asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml M asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml A asterixdb/asterix-server/src/main/opt/aws/bin/gen_volumes.sh M asterixdb/asterix-server/src/main/opt/aws/bin/start.sh M asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml M asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml M asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml M asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml 8 files changed, 112 insertions(+), 49 deletions(-) Approvals: Ian Maxon: Looks good to me, approved Jenkins: Verified; No violations found; No violations found diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml index 13de7cd..d6661ca 100644 --- a/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/instance_start.yml @@ -19,7 +19,7 @@ - hosts: [localhost,] vars: - temp_dir: "/tmp/asterixdb" + temp_dir: "{{ playbook_dir }}/../conf/instance" inventory: "{{ temp_dir }}/inventory" ccconf: "{{ temp_dir }}/cc.conf" tasks: diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml index 9d371d2..66fd79c 100644 --- a/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml +++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/start_cc.yml @@ -19,7 +19,7 @@ - name: Copy cluster config to CC synchronize: - src: /tmp/asterixdb/cc.conf + src: ../conf/instance/cc.conf dest: "{{ binarydir }}/cc.conf" - name: Update cluster config diff --git a/asterixdb/asterix-server/src/main/opt/aws/bin/gen_volumes.sh b/asterixdb/asterix-server/src/main/opt/aws/bin/gen_volumes.sh new file mode 100755 index 0000000..82ab685 --- /dev/null +++ b/asterixdb/asterix-server/src/main/opt/aws/bin/gen_volumes.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ + +# Number of instance-store volumes +count=$1 + +# Destination of the generated file for the volume list +dest=$2 + +# There is no instance-stores. +if [ $count -le 0 ] +then + printf "vols: []\n" >$dest + exit 0 +fi + +# Generates a list of instance-local volumes. +printf "vols:\n" >$dest + +for i in $(seq 1 $count) +do + printf ' - { device_name: /dev/xvd%c, ephemeral: ephemeral%d }\n' `printf "\x$(printf %x \`expr 97 + $i\`)"` \ + $(($i-1)) >> $dest +done diff --git a/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh b/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh index a836b4e..8bffaa4 100755 --- a/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh +++ b/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh @@ -30,7 +30,7 @@ ansible-playbook -i "localhost," $AWS_PATH/yaml/aws_start.yml # Generates an Ansible inventory file and an AsterixDB configuration file. -temp=/tmp/asterixdb +temp=$AWS_PATH/conf/instance inventory=$temp/inventory # Installs asterixdb on all AWS instances. diff --git a/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml index e0a060e..72b3d63 100644 --- a/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml +++ b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_types.yml @@ -1,37 +1,50 @@ -one_ephemeral: - - { device_name: /dev/xvdb, ephemeral: ephemeral0 } -two_ephemeral: - - { device_name: /dev/xvdb, ephemeral: ephemeral0 } - - { device_name: /dev/xvdc, ephemeral: ephemeral1 } -three_ephemeral: - - { device_name: /dev/xvdb, ephemeral: ephemeral0 } - - { device_name: /dev/xvdc, ephemeral: ephemeral1 } - - { device_name: /dev/xvdd, ephemeral: ephemeral2 } -four_ephemeral: - - { device_name: /dev/xvdb, ephemeral: ephemeral0 } - - { device_name: /dev/xvdc, ephemeral: ephemeral1 } - - { device_name: /dev/xvdd, ephemeral: ephemeral2 } - - { device_name: /dev/xvde, ephemeral: ephemeral3 } +# ------------------------------------------------------------ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ------------------------------------------------------------ ephemeral_volumes: - c1.medium: "{{ one_ephemeral }}" - c1.xlarge: "{{ four_ephemeral }}" - c3.large: "{{ two_ephemeral }}" - c3.xlarge: "{{ two_ephemeral }}" - c3.2xlarge: "{{ two_ephemeral }}" - c3.4xlarge: "{{ two_ephemeral }}" - c3.8xlarge: "{{ two_ephemeral }}" - i2.xlarge: "{{ one_ephemeral }}" - i2.2xlarge: "{{ two_ephemeral }}" - i2.4xlarge: "{{ four_ephemeral }}" - m1.small: "{{ one_ephemeral }}" - m1.medium: "{{ one_ephemeral }}" - m1.large: "{{ two_ephemeral }}" - m1.xlarge: "{{ four_ephemeral }}" - m2.xlarge: "{{ one_ephemeral }}" - m2.2xlarge: "{{ one_ephemeral }}" - m2.4xlarge: "{{ two_ephemeral }}" - m3.medium: "{{ one_ephemeral }}" - m3.large: "{{ one_ephemeral }}" - m3.xlarge: "{{ two_ephemeral }}" - m3.2xlarge: "{{ two_ephemeral }}" + c3.large: 2 + c3.xlarge: 2 + c3.2xlarge: 2 + c3.4xlarge: 2 + c3.8xlarge: 2 + d2.xlarge: 3 + d2.2xlarge: 6 + d2.4xlarge: 12 + d2.8xlarge: 24 + i3.large: 1 + i3.xlarge: 1 + i3.2xlarge: 1 + i3.4xlarge: 2 + i3.8xlarge: 4 + i3.16xlarge: 8 + f1.2xlarge: 1 + f1.16xlarge: 4 + g2.2xlarge: 1 + g2.8xlarge: 2 + m3.medium: 1 + m3.large: 1 + m3.xlarge: 2 + m3.2xlarge: 2 + r3.large: 1 + r3.xlarge: 1 + r3.2xlarge: 1 + r3.4xlarge: 1 + r3.8xlarge: 2 + x1.16xlarge: 1 + x1.32xlarge: 2 diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml index 344bfd5..a35cf12 100644 --- a/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/aws_start.yml @@ -21,17 +21,31 @@ hosts: localhost gather_facts: false vars: - temp_dir: "/tmp/asterixdb" + temp_dir: "{{ playbook_dir }}/../conf/instance" + bin_dir: "{{ playbook_dir }}/../bin" inventory: "{{ temp_dir }}/inventory" ccconf: "{{ temp_dir }}/cc.conf" home_dir: "/home/{{ user }}" - vols: "{{ ephemeral_volumes[instance_type]|default([]) }}" + num_vols: "{{ ephemeral_volumes[instance_type]|default(0) }}" tasks: - include_vars: ../conf/aws_settings.yml - include_vars: ../conf/aws_types.yml - name: Clean the temporary directory - shell: rm -rf "{{ temp_dir }}" + file: + path: "{{ temp_dir }}" + state: absent + + - name: Create local temporary directory + file: + path: "{{ temp_dir }}" + state: directory + + - name: Generate instance-store volumes for AWS instances + shell: "{{ playbook_dir }}/../bin/gen_volumes.sh {{ num_vols }} {{ temp_dir }}/volumes.yml" + + - name: Load generated volumes + include_vars: "{{ temp_dir }}/volumes.yml" - name: Launch all instances ec2: @@ -48,11 +62,6 @@ Name: "{{ tag }}" volumes: "{{ vols }}" register: ec2 - - - name: Create local temporary directory - file: - path: "{{ temp_dir }}" - state: directory - name: Set CC node set_fact: @@ -107,7 +116,7 @@ when: vols|length == 0 - name: Generates multiple iodevice paths - shell: printf "devices:\n" >> "{{ temp_dir }}/devices" + shell: printf "devices:\n" > "{{ temp_dir }}/devices" when: vols|length > 0 - name: Populate multiple iodevices diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml index cf09b69..3a9fd5a 100644 --- a/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/instance_start.yml @@ -25,7 +25,7 @@ - hosts: all tasks: - include_vars: ../conf/aws_settings.yml - - include_vars: ../conf/aws_types.yml + - include_vars: ../conf/instance/volumes.yml - name: Mount instance local stores include: mount.yml diff --git a/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml b/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml index 2e16476..03a740b 100644 --- a/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml +++ b/asterixdb/asterix-server/src/main/opt/aws/yaml/mount.yml @@ -23,6 +23,6 @@ sudo mkdir "/data{{ vol.0 + 1 }}" sudo mount "{{ vol.1.device_name }}" "/data{{ vol.0 + 1 }}" sudo chown -R $USER "/data{{ vol.0 + 1 }}" - with_indexed_items: "{{ ephemeral_volumes[instance_type]|default([]) }}" + with_indexed_items: "{{ vols }}" loop_control: loop_var: vol -- To view, visit https://asterix-gerrit.ics.uci.edu/1603 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I16efff22a029e316c7a7dc0402c0131e0c0bdea4 Gerrit-PatchSet: 5 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Yingyi Bu <buyin...@gmail.com> Gerrit-Reviewer: Ian Maxon <ima...@apache.org> Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Gerrit-Reviewer: Michael Blow <mb...@apache.org> Gerrit-Reviewer: Yingyi Bu <buyin...@gmail.com>
