This is an automated email from the ASF dual-hosted git repository.
arvindsh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/main by this push:
new 2977334 Update Azure specific Ansible modules (#434)
2977334 is described below
commit 2977334329fcecf5ed91e167da11d6b003f8dbcc
Author: Arvind Shyamsundar <arvin...@microsoft.com>
AuthorDate: Wed Nov 8 07:35:50 2023 -0800
Update Azure specific Ansible modules (#434)
* Update azure.azcollection to 1.18.1.
* Update Azure specific Ansible files to suit the current
azure.azcollection modules.
* Update the Azure specific Python code to use the current
credential method (see
[MSAL-based Azure
CLI](https://learn.microsoft.com/en-us/cli/azure/msal-based-azure-cli) if
interested to know mroe).
* Replace the custom azure_rm_virtualmachinescaleset_nic_list_facts with
the built-in azure.azcollection.azure_rm_vmssnetworkinterface_info.
* Update the options to use Azure Spot vs. regular VMs to `Spot` and
`None` as per the current definitions in the Azure APIs.
---
README.md | 11 +-
ansible/azure_terminate.yml | 24 +-
...ure_rm_virtualmachinescaleset_nic_list_facts.py | 286 ---------------------
.../roles/azure/tasks/create_common_resources.yml | 4 +-
ansible/roles/azure/tasks/create_multiple_vmss.yml | 16 +-
ansible/roles/azure/tasks/create_vmss.yml | 16 +-
ansible/roles/common/tasks/ssh.yml | 4 +-
ansible/roles/hadoop/tasks/main.yml | 2 +-
ansible/roles/proxy/tasks/main.yml | 2 +-
conf/azure_multiple_vmss_vars.yml.example | 2 +-
conf/muchos.props.example | 8 +-
docs/azure-multiple-vmss.md | 2 +-
lib/muchos/azure.py | 9 +-
lib/muchos/config/azure.py | 7 +-
lib/muchos/config/azurevalidations.py | 19 +-
lib/muchos/config/base.py | 20 +-
lib/muchos/config/ec2.py | 4 +-
lib/muchos/ec2.py | 4 +-
lib/muchos/existing.py | 8 +-
lib/tests/azure/test_config.py | 2 +-
lib/tests/ec2/test_config.py | 2 +-
scripts/cibuild | 2 +-
scripts/install-ansible-collections | 2 +-
scripts/install-ansible-for-azure | 5 +-
24 files changed, 95 insertions(+), 366 deletions(-)
diff --git a/README.md b/README.md
index 38600c0..f7d982e 100644
--- a/README.md
+++ b/README.md
@@ -61,15 +61,7 @@ Muchos requires the following for EC2 installations:
Muchos requires the following for Azure installations:
* [Azure
CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest)
must be installed,
- configured and authenticated to an Azure subscription. Please note - you
should install
- [Azure CLI
2.0.69](https://packages.microsoft.com/yumrepos/azure-cli/azure-cli-2.0.69-1.el7.x86_64.rpm)
on CentOS.
- Higher versions of Azure CLI are unsupported for Muchos on CentOS at this
time until
- [this issue](https://github.com/Azure/azure-cli/issues/10128) in the Azure
CLI 2.0.70 is fixed.
- Example command to install Azure CLI 2.0.69 on CentOS is below:
-```bash
-wget
https://packages.microsoft.com/yumrepos/azure-cli/azure-cli-2.0.69-1.el7.x86_64.rpm
-sudo yum install azure-cli-2.0.69-1.el7.x86_64.rpm
-```
+ configured and authenticated to an Azure subscription. It is recommended to
use Azure CLI 2.50 or later.
* An Azure account with permissions to either use an existing or create new
Resource Groups, Virtual Networks and Subnets
* A machine which can connect to securely deploy the cluster in Azure.
* Install the [Ansible collection for
Azure](https://github.com/ansible-collections/azure), and associated
pre-requisites within
@@ -165,6 +157,7 @@ Under the `general` section, edit following values as per
your configuration
* `proxy_hostname` (optional) is the name of the machine which has access to
the cluster VNET
Under the `azure` section, edit following values as per your configuration:
+* `azure_subscription_id` to provide the Azure subscription GUID
* `resource_group` to provide the resource-group name for the cluster
deployment. A new resource group with
this name will be created if it doesn't already exist
* `vnet` to provide the name of the VNET that your cluster nodes should use. A
new VNET with this name will be
diff --git a/ansible/azure_terminate.yml b/ansible/azure_terminate.yml
index 048bcd1..a5ae7bc 100644
--- a/ansible/azure_terminate.yml
+++ b/ansible/azure_terminate.yml
@@ -25,7 +25,7 @@
- import_tasks: roles/azure/tasks/application_insights_common.yml
- name: Delete application insights
- azure_rm_resource:
+ azure.azcollection.azure_rm_resource:
resource_group: "{{ resource_group }}"
provider: insights
resource_type: components
@@ -34,7 +34,7 @@
state: absent
- name: Delete workbook
- azure_rm_resource:
+ azure.azcollection.azure_rm_resource:
resource_group: "{{ resource_group }}"
provider: insights
resource_type: workbooks
@@ -44,7 +44,7 @@
when: workbook_exists
- name: Delete dashboard
- azure_rm_resource:
+ azure.azcollection.azure_rm_resource:
resource_group: "{{ resource_group }}"
provider: Portal
resource_type: dashboards
@@ -53,7 +53,7 @@
state: absent
- name: Delete log analytics workspace
- azure_rm_resource:
+ azure.azcollection.azure_rm_resource:
resource_group: "{{ resource_group }}"
provider: OperationalInsights
resource_type: workspaces
@@ -62,7 +62,7 @@
state: absent
- name: Delete User Assigned Identity
- azure_rm_resource:
+ azure.azcollection.azure_rm_resource:
resource_group: "{{ resource_group }}"
provider: ManagedIdentity
resource_type: userAssignedIdentities
@@ -72,7 +72,7 @@
when: use_adlsg2
- name: Delete ADLS Gen2 storage Account
- azure_rm_storageaccount:
+ azure.azcollection.azure_rm_storageaccount:
resource_group: "{{ resource_group }}"
name: "{{ item.split('@')[1].split('.')[0] }}"
force_delete_nonempty: yes
@@ -81,7 +81,7 @@
when: use_adlsg2
- name: Delete VM Scale Set
- azure_rm_virtualmachinescaleset:
+ azure.azcollection.azure_rm_virtualmachinescaleset:
resource_group: "{{ resource_group }}"
name: "{{ vmss_name }}"
remove_on_absent: all
@@ -95,7 +95,7 @@
when: use_multiple_vmss
- name: Delete VM Scale Sets
- azure_rm_virtualmachinescaleset:
+ azure.azcollection.azure_rm_virtualmachinescaleset:
resource_group: "{{ resource_group }}"
name: "{{ vmss_name }}-{{ item.name_suffix }}"
remove_on_absent: all
@@ -105,7 +105,7 @@
when: use_multiple_vmss
- name: Delete azure proxy virtual machine if one was created
- azure_rm_virtualmachine:
+ azure.azcollection.azure_rm_virtualmachine:
resource_group: "{{ resource_group }}"
name: "{{ azure_proxy_host }}"
remove_on_absent: all
@@ -113,20 +113,20 @@
when: azure_proxy_host is defined and azure_proxy_host|length
- name: Delete corresponding network security group
- azure_rm_securitygroup:
+ azure.azcollection.azure_rm_securitygroup:
resource_group: "{{ resource_group }}"
name: "{{ azure_proxy_host }}-nsg"
state: absent
when: azure_proxy_host is defined and azure_proxy_host|length
- name: Get facts for virtual network
- azure_rm_virtualnetwork_info:
+ azure.azcollection.azure_rm_virtualnetwork_info:
resource_group: "{{ resource_group }}"
name: "{{ vnet }}"
register: vnet_facts
- name: Delete corresponding virtual network if one was created
- azure_rm_virtualnetwork:
+ azure.azcollection.azure_rm_virtualnetwork:
resource_group: "{{ resource_group }}"
name: "{{ vnet }}"
state: absent
diff --git a/ansible/library/azure_rm_virtualmachinescaleset_nic_list_facts.py
b/ansible/library/azure_rm_virtualmachinescaleset_nic_list_facts.py
deleted file mode 100644
index e320251..0000000
--- a/ansible/library/azure_rm_virtualmachinescaleset_nic_list_facts.py
+++ /dev/null
@@ -1,286 +0,0 @@
-#!/usr/bin/python3
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# flake8: noqa
-from __future__ import absolute_import, division, print_function
-__metaclass__ = type
-
-import six
-
-
-ANSIBLE_METADATA = {'metadata_version': '1.1',
- 'status': ['preview'],
- 'supported_by': 'community'}
-
-
-DOCUMENTATION = '''
----
-module: azure_rm_virtualmachinescaleset_nic_list_facts
-
-version_added: "2.9"
-
-short_description: Get network interface facts for a VMSS
-
-description:
- - Get a list of all network interfaces for a virtual machine scale set
-
-options:
- vmss_name:
- description:
- - Name of the virtual machine scale set.
- required: true
- resource_group:
- description:
- - Name of the resource group containing the virtual machine scale
set.
- required: true
-
-extends_documentation_fragment:
- - azure
-
-'''
-
-EXAMPLES = '''
- - name: Get all network interfaces in a virtual machine scale set
- azure_rm_virtualmachinescaleset_nic_list_facts:
- resource_group: myResourceGroup
- vmss_name: myvmss
-'''
-
-RETURN = '''
-azure_networkinterfaces:
- description: List of network interface dicts.
- returned: always
- type: list
- example: [{
- "dns_settings": {
- "applied_dns_servers": [],
- "dns_servers": [],
- "internal_dns_name_label": null,
- "internal_fqdn": null
- },
- "enable_ip_forwarding": false,
- "etag": 'W/"59726bfc-08c4-44ed-b900-f6a559876a9d"',
- "id":
"/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/resourceGroup/myResourceGroup/providers/Microsoft.Network/networkInterfaces/nic003",
- "ip_configuration": {
- "name": "default",
- "private_ip_address": "10.10.0.4",
- "private_ip_allocation_method": "Dynamic",
- "public_ip_address": {
- "id":
"/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/resourceGroup/myResourceGroup/providers/Microsoft.Network/publicIPAddresses/publicip001",
- "name": "publicip001"
- },
- "subnet": {
- "id":
"/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/resourceGroup/myResourceGroup/providers/Microsoft.Network/virtualNetworks/vnet001/subnets/subnet001",
- "name": "subnet001",
- "virtual_network_name": "vnet001"
- }
- },
- "location": "westus",
- "mac_address": null,
- "name": "nic003",
- "network_security_group": {
- "id":
"/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/resourceGroup/myResourceGroup/providers/Microsoft.Network/networkSecurityGroups/secgroup001",
- "name": "secgroup001"
- },
- "primary": null,
- "provisioning_state": "Succeeded",
- "tags": {},
- "type": "Microsoft.Network/networkInterfaces"
- }]
-networkinterfaces:
- description: List of network interface dict, the dict contains parameters
can be passed to C(azure_rm_networkinterface) module.
- type: list
- returned: always
- contains:
- id:
- description:
- - Id of the network interface.
- resource_group:
- description:
- - Name of a resource group where the network interface exists.
- name:
- description:
- - Name of the network interface.
- location:
- description:
- - Azure location.
- virtual_network:
- description:
- - An existing virtual network with which the network interface
will be associated.
- - It is a dict which contains C(name) and C(resource_group) of
the virtual network.
- subnet:
- description:
- - Name of an existing subnet within the specified virtual
network.
- tags:
- description:
- - Tags of the network interface.
- ip_configurations:
- description:
- - List of ip configuration if contains mutilple configuration.
- contains:
- name:
- description:
- - Name of the ip configuration.
- private_ip_address:
- description:
- - Private ip address for the ip configuration.
- private_ip_allocation_method:
- description:
- - private ip allocation method.
- public_ip_address:
- description:
- - Name of the public ip address. None for disable ip
address.
- public_ip_allocation_method:
- description:
- - public ip allocation method.
- load_balancer_backend_address_pools:
- description:
- - List of an existing load-balancer backend address
pool id to associate with the network interface.
- primary:
- description:
- - Whether the ip configuration is the primary one in
the list.
- application_security_groups:
- description:
- - List of Application security groups.
- sample:
/subscriptions/<subsid>/resourceGroups/<rg>/providers/Microsoft.Network/applicationSecurityGroups/myASG
- enable_accelerated_networking:
- description:
- - Specifies whether the network interface should be created
with the accelerated networking feature or not
- create_with_security_group:
- description:
- - Specifies whether a default security group should be be
created with the NIC. Only applies when creating a new NIC.
- type: bool
- security_group:
- description:
- - A security group resource ID with which to associate the
network interface.
- enable_ip_forwarding:
- description:
- - Whether to enable IP forwarding
- dns_servers:
- description:
- - Which DNS servers should the NIC lookup
- - List of IP's
- mac_address:
- description:
- - The MAC address of the network interface.
- provisioning_state:
- description:
- - The provisioning state of the network interface.
- dns_settings:
- description:
- - The DNS settings in network interface.
- contains:
- dns_servers:
- description: List of DNS servers IP addresses.
- applied_dns_servers:
- description:
- - If the VM that uses this NIC is part of an
Availability Set, then this list will have the union of all DNS servers
- from all NICs that are part of the Availability Set.
This property is what is configured on each of those VMs.
- internal_dns_name_label:
- description: Relative DNS name for this NIC used for
internal communications between VMs in the same virtual network.
- internal_fqdn:
- description: Fully qualified DNS name supporting internal
communications between VMs in the same virtual network.
-''' # NOQA
-try:
- from msrestazure.azure_exceptions import CloudError
- from azure.common import AzureMissingResourceHttpError, AzureHttpError
-except Exception:
- # This is handled in azure_rm_common
- pass
-
-from
ansible_collections.azure.azcollection.plugins.module_utils.azure_rm_common
import AzureRMModuleBase, azure_id_to_dict
-
-
-AZURE_OBJECT_CLASS = 'NetworkInterface'
-
-
-class AzureRMVirtualMachineScaleSetNetworkInterfaceFacts(AzureRMModuleBase):
-
- def __init__(self):
-
- self.module_arg_spec = dict(
- vmss_name=dict(type='str', required=True),
- resource_group=dict(type='str', required=True),
- tags=dict(type='list')
- )
-
- self.results = dict(
- changed=False,
- networkinterfaces=[]
- )
-
- self.vmss_name = None
- self.resource_group = None
- self.tags = None
-
- super(AzureRMVirtualMachineScaleSetNetworkInterfaceFacts,
- self).__init__(self.module_arg_spec,
- supports_tags=False,
- facts_module=True
- )
-
- def exec_module(self, **kwargs):
-
- for key in self.module_arg_spec:
- setattr(self, key, kwargs[key])
-
- if not (self.vmss_name and self.resource_group):
- self.fail("Parameter error: resource group and name are required
parameters.")
-
- networkinterfaces = []
-
- networkinterfaces = self.list_nics()
-
- self.results['networkinterfaces'] =
self._promote_properties(self.serialize_nics(networkinterfaces))
- return self.results
-
- def list_nics(self):
- self.log('Get properties for {0}'.format(self.vmss_name))
- item = None
- try:
- return
self.network_client.network_interfaces.list_virtual_machine_scale_set_network_interfaces(self.resource_group,
self.vmss_name)
- except Exception as exc:
- self.fail("Error fetching nic list - {0}".format(str(exc)))
-
-
- def serialize_nics(self, raws):
- return [self.serialize_obj(item, AZURE_OBJECT_CLASS) for item in raws]
if raws else []
-
-
- def _promote_properties(self, obj):
- if isinstance(obj, list):
- return [self._promote_properties(i) for i in obj]
-
- if isinstance(obj, dict):
- for k,v in six.iteritems(obj):
- if isinstance(v, (list, dict)):
- obj.update({k: self._promote_properties(v)})
-
- if isinstance(obj.get("properties"), dict):
- properties = obj.pop("properties")
- obj.update(properties)
-
- return obj
-
-
-def main():
- AzureRMVirtualMachineScaleSetNetworkInterfaceFacts()
-
-
-if __name__ == '__main__':
- main()
diff --git a/ansible/roles/azure/tasks/create_common_resources.yml
b/ansible/roles/azure/tasks/create_common_resources.yml
index 745f929..71c9f70 100644
--- a/ansible/roles/azure/tasks/create_common_resources.yml
+++ b/ansible/roles/azure/tasks/create_common_resources.yml
@@ -39,13 +39,13 @@
retries: 5
delay: 3
register: resource_group_facts
- until: resource_group_facts.resourcegroups is defined and
resource_group_facts.resourcegroups|map(attribute='name')|join('') is defined
and
resource_group_facts.resourcegroups|map(attribute='properties')|map(attribute='provisioningState')|join('')
== "Succeeded"
+ until: resource_group_facts.resourcegroups is defined and
resource_group_facts.resourcegroups|map(attribute='name')|join('') is defined
and
resource_group_facts.resourcegroups|map(attribute='properties')|map(attribute='provisioning_state')|join('')
== "Succeeded"
- debug:
msg: "Name of resource group created: {{
resource_group_facts.resourcegroups|map(attribute='name')|join('') }}"
- debug:
- msg: "Provisioning state of {{ resource_group }} is {{
resource_group_facts.resourcegroups|map(attribute='properties')|map(attribute='provisioningState')|join('')
}}"
+ msg: "Provisioning state of {{ resource_group }} is {{
resource_group_facts.resourcegroups|map(attribute='properties')|map(attribute='provisioning_state')|join('')
}}"
- name: Create a virtual network
azure_rm_virtualnetwork:
diff --git a/ansible/roles/azure/tasks/create_multiple_vmss.yml
b/ansible/roles/azure/tasks/create_multiple_vmss.yml
index cec2d13..16c5a06 100644
--- a/ansible/roles/azure/tasks/create_multiple_vmss.yml
+++ b/ansible/roles/azure/tasks/create_multiple_vmss.yml
@@ -31,12 +31,12 @@
name: azure_multiple_vmss_vars
- name: Create Scale Set
- community.azure.azure_rm_virtualmachinescaleset:
+ azure.azcollection.azure_rm_virtualmachinescaleset:
resource_group: "{{ resource_group }}"
location: "{{ location }}"
name: "{{ vmss_name }}-{{ item.name_suffix }}"
vm_size: "{{ item.sku }}"
- priority: "{{ item.vmss_priority | default('Regular') }}"
+ priority: "{{ item.vmss_priority | default('None') }}"
admin_username: "{{ cluster_user }}"
ssh_password_enabled: false
ssh_public_keys:
@@ -99,7 +99,7 @@
poll: 0
- name: "Get VMSS nic list"
- azure_rm_virtualmachinescaleset_nic_list_facts:
+ azure.azcollection.azure_rm_vmssnetworkinterface_info:
resource_group: "{{ resource_group }}"
vmss_name: "{{ vmss_name }}-{{ item.name_suffix }}"
register: _vmss_nic_list
@@ -126,6 +126,9 @@
retries: 300
with_items: "{{ _vmss_nic_list.results }}"
+# Construct a dictionary of desired host names to their IP addresses
+# When doing this, replace any underscore characters in the Azure-generated VM
instance name
+# with a dash and use the first 12 "portions" of the NIC resource ID to get
the VM resource ID
- name: Get VM hostname to IP mapping
set_fact:
hostname_ip_pairs: |
@@ -141,9 +144,10 @@
{%- endfor -%}
{%- set vmid_ips = {} -%}
{%- for vmss in vmss_nic_list.results -%}
- {%- for interface in vmss.networkinterfaces -%}
- {%- if interface.virtualMachine is defined -%}
- {%- set _ = vmid_ips.__setitem__(interface.virtualMachine.id,
interface.ipConfigurations[0].privateIPAddress) -%}
+ {%- for interface in vmss.vmss_networkinterfaces -%}
+ {%- if interface.id is defined -%}
+ {%- set vmid = "/".join(interface.id.split("/")[0:11]) -%}
+ {%- set _ = vmid_ips.__setitem__(vmid,
interface.ip_configurations[0].private_ip_address) -%}
{%- endif -%}
{%- endfor -%}
{%- endfor -%}
diff --git a/ansible/roles/azure/tasks/create_vmss.yml
b/ansible/roles/azure/tasks/create_vmss.yml
index e819db7..8b79625 100644
--- a/ansible/roles/azure/tasks/create_vmss.yml
+++ b/ansible/roles/azure/tasks/create_vmss.yml
@@ -39,12 +39,12 @@
when: numnodes > 100
- name: Create Scale Set
- community.azure.azure_rm_virtualmachinescaleset:
+ azure.azcollection.azure_rm_virtualmachinescaleset:
resource_group: "{{ resource_group }}"
location: "{{ location }}"
name: "{{ vmss_name }}"
vm_size: "{{ vm_sku }}"
- priority: "{{ vmss_priority | default('Regular') }}"
+ priority: "{{ vmss_priority | default('None') }}"
admin_username: "{{ cluster_user }}"
ssh_password_enabled: false
ssh_public_keys:
@@ -94,11 +94,14 @@
recurse: yes
- name: "Get VMSS nic list"
- azure_rm_virtualmachinescaleset_nic_list_facts:
+ azure.azcollection.azure_rm_vmssnetworkinterface_info:
resource_group: "{{ resource_group }}"
vmss_name: "{{ vmss_name }}"
register: vmss_nic_list
+# Construct a dictionary of desired host names to their IP addresses
+# When doing this, replace any underscore characters in the Azure-generated VM
instance name
+# with a dash and use the first 12 "portions" of the NIC resource ID to get
the VM resource ID
- name: Get VM hostname to IP mapping
set_fact:
hostname_ip_pairs: |
@@ -111,9 +114,10 @@
{%- set _ = vmid_names.__setitem__(instance.id,
instance.name.replace('_','-')) -%}
{%- endfor -%}
{%- set vmid_ips = {} -%}
- {%- for interface in vmss_nic_list.networkinterfaces -%}
- {%- if interface.virtualMachine is defined -%}
- {%- set _ = vmid_ips.__setitem__(interface.virtualMachine.id,
interface.ipConfigurations[0].privateIPAddress) -%}
+ {%- for interface in vmss_nic_list.vmss_networkinterfaces -%}
+ {%- if interface.id is defined -%}
+ {%- set vmid = "/".join(interface.id.split("/")[0:11]) -%}
+ {%- set _ = vmid_ips.__setitem__(vmid,
interface.ip_configurations[0].private_ip_address) -%}
{%- endif -%}
{%- endfor -%}
{%- for vmid in vmid_names -%}
diff --git a/ansible/roles/common/tasks/ssh.yml
b/ansible/roles/common/tasks/ssh.yml
index ecb0914..ed03613 100644
--- a/ansible/roles/common/tasks/ssh.yml
+++ b/ansible/roles/common/tasks/ssh.yml
@@ -22,8 +22,8 @@
- name: "copy cluster public key to all nodes"
copy: src=/home/{{ cluster_user }}/.ssh/id_rsa.pub dest=/home/{{
cluster_user }}/.ssh/id_rsa.pub owner={{ cluster_user }} group={{ cluster_group
}} mode="0644"
- name: "add cluster user to ~/.ssh/authorized_keys"
- ansible.posix.authorized_key: user={{ cluster_user }} key="{{ lookup('file',
'/home/' + cluster_user + '/.ssh/id_rsa.pub') }}"
+ authorized_key: user={{ cluster_user }} key="{{ lookup('file', '/home/' +
cluster_user + '/.ssh/id_rsa.pub') }}"
- name: "add conf/keys to ~/.ssh/authorized_keys"
- ansible.posix.authorized_key: user={{ cluster_user }} key="{{ lookup('file',
'conf/keys') }}"
+ authorized_key: user={{ cluster_user }} key="{{ lookup('file', 'conf/keys')
}}"
- name: "set ssh config"
copy: src=roles/common/files/ssh_config dest=/home/{{ cluster_user
}}/.ssh/config owner={{ cluster_user }} group={{ cluster_group }} mode=0600
diff --git a/ansible/roles/hadoop/tasks/main.yml
b/ansible/roles/hadoop/tasks/main.yml
index 445a731..2126c8c 100644
--- a/ansible/roles/hadoop/tasks/main.yml
+++ b/ansible/roles/hadoop/tasks/main.yml
@@ -38,7 +38,7 @@
# This is currently needed to run hadoop with Java 11 (see
https://github.com/apache/fluo-muchos/issues/266)
- name: "Copy javax.activation-api (when Hadoop 3 and Java 11 are used)"
- ansible.posix.synchronize: src={{ user_home }}/mvn_dep/ dest={{ hadoop_home
}}/share/hadoop/common/lib/
+ synchronize: src={{ user_home }}/mvn_dep/ dest={{ hadoop_home
}}/share/hadoop/common/lib/
when: hadoop_major_version == '3' and java_product_version == 11
- name: "setup hadoop short circuit socket dir"
diff --git a/ansible/roles/proxy/tasks/main.yml
b/ansible/roles/proxy/tasks/main.yml
index 7eb7253..bcb45fa 100644
--- a/ansible/roles/proxy/tasks/main.yml
+++ b/ansible/roles/proxy/tasks/main.yml
@@ -16,7 +16,7 @@
#
- name: "add conf/keys to ~/.ssh/authorized_keys"
- ansible.posix.authorized_key: user={{ cluster_user }} key="{{ lookup('file',
'conf/keys') }}"
+ authorized_key: user={{ cluster_user }} key="{{ lookup('file', 'conf/keys')
}}"
tags: configure_os
- name: "ensure cluster user exists and generate ssh key"
user: name={{ cluster_user }} generate_ssh_key=yes ssh_key_bits=4096
state=present
diff --git a/conf/azure_multiple_vmss_vars.yml.example
b/conf/azure_multiple_vmss_vars.yml.example
index 5d2f964..8d6d7d2 100644
--- a/conf/azure_multiple_vmss_vars.yml.example
+++ b/conf/azure_multiple_vmss_vars.yml.example
@@ -74,7 +74,7 @@ vars_list:
# As an example, the below is also configured as an Azure Spot
(Low-Priority) VMSS
- name_suffix: vmss6
sku: Standard_D8s_v3
- vmss_priority: Low
+ vmss_priority: Spot
perf_profile: azd8s
data_disk_count: 8
data_disk_sku: Standard_LRS
diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index f25a266..4d59a65 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -104,6 +104,8 @@ shutdown_delay_minutes = 0
shutdown_behavior = stop
[azure]
+# Azure subscription ID
+azure_subscription_id =
# Name of the Azure resource group to use. All resources are created within
this
# resource group. If a resource group with this name already exists, it will be
# used, otherwise a new resource group with this name will be created and used.
@@ -120,11 +122,11 @@ subnet_cidr = 10.1.0.0/16
#Optional. If set to True, will create multiple VMSS based on
multiple_vmss_vars.yml
use_multiple_vmss = False
# The below "vmss_priority" controls whether Azure Spot is used (or not).
-# When vmss_priority is set to Low, the cluster uses Azure Spot VM Scale Sets.
-# When vmss_priority is set to Regular (which is the default), Azure Spot is
NOT used.
+# When vmss_priority is set to Spot, the cluster uses Azure Spot VM Scale Sets.
+# When vmss_priority is set to None (which is the default), Azure Spot is NOT
used.
# Note that for multiple-VMSS deployments, this setting can be overridden at a
# per-VMSS level by specifying vmss_priority in the
conf/azure_multiple_vmss_vars.yml file.
-vmss_priority = Regular
+vmss_priority = None
# Azure image reference defined as a pipe-delimited string in the format
offer|publisher|sku|version|
# Please refer 'Launching an Azure cluster' section of the README before
making changes
azure_image_reference = CentOS|OpenLogic|7_9|latest|
diff --git a/docs/azure-multiple-vmss.md b/docs/azure-multiple-vmss.md
index 18d6818..7f9db02 100644
--- a/docs/azure-multiple-vmss.md
+++ b/docs/azure-multiple-vmss.md
@@ -22,7 +22,7 @@ Muchos provides a [sample
file](../conf/azure_multiple_vmss_vars.yml.example) wh
|-----------|------------------------|---------|-------------|
| `name_suffix` | Required | - | The name of each VMSS is constructed by
concatenating the Muchos cluster name with this string. As an example, if your
Muchos cluster is called `test`, and this field has a value of `ldr`, then the
VMSS is created with a name `test-ldr`|
| `sku` | Required | - | A string identifier specifying the Azure VM size.
Refer to the [Azure
documentation](https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series)
to lookup these strings. An example VM size is `Standard_D32s_v3` for a
32-vCPU
[Dsv3](https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series#dsv3-series)
VM|
-| `vmss_priority` | Optional | Regular | If this not specified at each VM
level, the value for `vmss_priority` from the `azure` section in
[muchos.props](../conf/muchos.props.example) is used | This can be set to
`Regular`, for regular VMs, or `Low` for [Spot
VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/windows/spot-vms).|
+| `vmss_priority` | Optional | None | If this not specified at each VM level,
the value for `vmss_priority` from the `azure` section in
[muchos.props](../conf/muchos.props.example) is used | This can be set to
`None`, for regular VMs, or `Spot` for [Spot
VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/windows/spot-vms).|
| `perf_profile` | Required | - | A string identifying a corresponding
performance profile configuration section in muchos.props which contains perf
profile parameters |
| `azure_disk_device_path`| Optional | If not specified, the corresponding
`azure_disk_device_path` value from the `azure` section in
[muchos.props](../conf/muchos.props.example) is used | This is a device path
used to enumerate attached SCSI or NVME disks to use for persistent local
storage |
| `azure_disk_device_pattern`| Optional | If not specified, the corresponding
`azure_disk_device_pattern` value from the `azure` section in
[muchos.props](../conf/muchos.props.example) is used | This is a device name
wildcard pattern used (internally) in conjunction with `azure_disk_device_path`
to enumerate attached SCSI or NVME disks to use for persistent local storage |
diff --git a/lib/muchos/azure.py b/lib/muchos/azure.py
index 54cdbb3..ef3e2b1 100644
--- a/lib/muchos/azure.py
+++ b/lib/muchos/azure.py
@@ -19,7 +19,7 @@
import json
import subprocess
from os import path
-from azure.common.client_factory import get_client_from_cli_profile
+from azure.identity import DefaultAzureCredential
from azure.mgmt.compute import ComputeManagementClient
from .existing import ExistingCluster
@@ -53,7 +53,10 @@ class VmssCluster(ExistingCluster):
def status(self):
config = self.config
azure_vars_dict = dict(config.items("azure"))
- compute_client = get_client_from_cli_profile(ComputeManagementClient)
+ credential = DefaultAzureCredential()
+ compute_client = ComputeManagementClient(
+ credential, azure_vars_dict["azure_subscription_id"]
+ )
vmss_status = compute_client.virtual_machine_scale_sets.get(
azure_vars_dict["resource_group"], self.config.cluster_name
)
@@ -177,7 +180,7 @@ class VmssCluster(ExistingCluster):
),
"w",
) as vmss_file:
- for (name, value) in self.config.items(profile):
+ for name, value in self.config.items(profile):
print("{0}: {1}".format(name, value), file=vmss_file)
# use VMSS-specific mount root if one is defined or
diff --git a/lib/muchos/config/azure.py b/lib/muchos/config/azure.py
index 6833426..f90810b 100644
--- a/lib/muchos/config/azure.py
+++ b/lib/muchos/config/azure.py
@@ -300,6 +300,11 @@ class AzureDeployConfig(BaseConfig):
def numnodes(self):
return self.getint("azure", "numnodes")
+ @ansible_host_var
+ @default(None)
+ def azure_subscription_id(self):
+ return self.get("azure", "azure_subscription_id")
+
@ansible_host_var
@default(None)
def resource_group(self):
@@ -341,7 +346,7 @@ class AzureDeployConfig(BaseConfig):
return self.get("azure", "azure_proxy_host_vm_sku")
@ansible_host_var
- @is_valid(is_in(["Regular", "Low"]))
+ @is_valid(is_in(["None", "Spot"]))
def vmss_priority(self):
return self.get("azure", "vmss_priority")
diff --git a/lib/muchos/config/azurevalidations.py
b/lib/muchos/config/azurevalidations.py
index cdbdc39..e248691 100644
--- a/lib/muchos/config/azurevalidations.py
+++ b/lib/muchos/config/azurevalidations.py
@@ -22,18 +22,23 @@ from .azurevalidationhelpers import (
vmss_exists,
)
from azure.mgmt.compute import ComputeManagementClient
-from azure.common.client_factory import get_client_from_cli_profile
+from azure.identity import DefaultAzureCredential
def validate_azure_configs(config, action):
+ credential = DefaultAzureCredential()
+
# get VM SKU resources for this location. we have to use
# a specific API version to do this as this resource_skus
# list operation is not allowed in any other API versions
# which are available with the version of Azure SDK
# that ships with Ansible for Azure
- config.client = get_client_from_cli_profile(
- ComputeManagementClient, api_version="2017-09-01"
+ config.client = ComputeManagementClient(
+ credential,
+ subscription_id=config.azure_subscription_id(),
+ api_version="2017-09-01",
)
+
config.vm_skus_for_location = list(
filter(
lambda s: s.resource_type == "virtualMachines"
@@ -44,8 +49,10 @@ def validate_azure_configs(config, action):
# switch to 2018-06-01 API which has support for other operations
# including VMSS checks
- config.client = get_client_from_cli_profile(
- ComputeManagementClient, api_version="2018-06-01"
+ config.client = ComputeManagementClient(
+ credential,
+ subscription_id=config.azure_subscription_id(),
+ api_version="2018-06-01",
)
validations = (
@@ -98,7 +105,7 @@ AZURE_VALIDATIONS = {
lambda config, client: config.getboolean(
"azure", "use_multiple_vmss"
)
- or not config.vmss_priority() == "Low"
+ or not config.vmss_priority() == "Spot"
or config.vm_sku() in config.spot_capable_skus(),
"azure.vm_sku must be an Azure Spot (low priority) capable VM SKU",
),
diff --git a/lib/muchos/config/base.py b/lib/muchos/config/base.py
index f7373a9..e015a7a 100644
--- a/lib/muchos/config/base.py
+++ b/lib/muchos/config/base.py
@@ -308,7 +308,7 @@ class BaseConfig(ConfigParser, metaclass=ABCMeta):
def _init_nodes(self):
self.node_d = {}
- for (hostname, value) in self.items("nodes"):
+ for hostname, value in self.items("nodes"):
if hostname in self.node_d:
exit(
"Hostname {0} already exists twice in nodes".format(
@@ -495,7 +495,7 @@ class BaseConfig(ConfigParser, metaclass=ABCMeta):
return self.node_d[hostname]
def has_service(self, service):
- for (hostname, service_list) in list(self.node_d.items()):
+ for hostname, service_list in list(self.node_d.items()):
if service in service_list:
return True
return False
@@ -503,7 +503,7 @@ class BaseConfig(ConfigParser, metaclass=ABCMeta):
# test method, might want to make private or just move to test module
def get_host_services(self):
retval = []
- for (hostname, service_list) in list(self.node_d.items()):
+ for hostname, service_list in list(self.node_d.items()):
retval.append((hostname, " ".join(service_list)))
retval.sort()
return retval
@@ -511,7 +511,7 @@ class BaseConfig(ConfigParser, metaclass=ABCMeta):
# test method, might want to make private or just move to test module
def get_service_private_ips(self, service):
retval = []
- for (hostname, service_list) in list(self.node_d.items()):
+ for hostname, service_list in list(self.node_d.items()):
if service in service_list:
retval.append(self.get_private_ip(hostname))
retval.sort()
@@ -519,7 +519,7 @@ class BaseConfig(ConfigParser, metaclass=ABCMeta):
def get_service_hostnames(self, service):
retval = []
- for (hostname, service_list) in list(self.node_d.items()):
+ for hostname, service_list in list(self.node_d.items()):
if service in service_list:
retval.append(hostname)
retval.sort()
@@ -529,7 +529,7 @@ class BaseConfig(ConfigParser, metaclass=ABCMeta):
def get_non_proxy(self):
retval = []
proxy_ip = self.get_private_ip(self.get("general", "proxy_hostname"))
- for (hostname, (private_ip, public_ip)) in list(
+ for hostname, (private_ip, public_ip) in list(
self.get_hosts().items()
):
if private_ip != proxy_ip:
@@ -539,7 +539,7 @@ class BaseConfig(ConfigParser, metaclass=ABCMeta):
def get_private_ip_hostnames(self):
retval = []
- for (hostname, (private_ip, public_ip)) in list(
+ for hostname, (private_ip, public_ip) in list(
self.get_hosts().items()
):
retval.append((private_ip, hostname))
@@ -617,13 +617,13 @@ class BaseConfig(ConfigParser, metaclass=ABCMeta):
def print_all(self):
print("proxy_public_ip = ", self.proxy_public_ip())
- for (name, val) in self.items("general"):
+ for name, val in self.items("general"):
print(name, "=", val)
- for (name, val) in self.items("ec2"):
+ for name, val in self.items("ec2"):
print(name, "=", val)
- for (name, val) in self.items("azure"):
+ for name, val in self.items("azure"):
print(name, "=", val)
def print_property(self, key):
diff --git a/lib/muchos/config/ec2.py b/lib/muchos/config/ec2.py
index c2853f9..664723a 100644
--- a/lib/muchos/config/ec2.py
+++ b/lib/muchos/config/ec2.py
@@ -61,7 +61,7 @@ class Ec2DeployConfig(BaseConfig):
def init_nodes(self):
self.node_d = {}
- for (hostname, value) in self.items("nodes"):
+ for hostname, value in self.items("nodes"):
if hostname in self.node_d:
exit(
"Hostname {0} already exists twice in nodes".format(
@@ -105,7 +105,7 @@ class Ec2DeployConfig(BaseConfig):
("worker", self.worker_ephemeral_devices()),
]
- for (ntype, devices) in node_list:
+ for ntype, devices in node_list:
node_types[ntype] = {
"mounts": self.mounts(len(devices)),
"devices": devices,
diff --git a/lib/muchos/ec2.py b/lib/muchos/ec2.py
index 695975e..23e82eb 100644
--- a/lib/muchos/ec2.py
+++ b/lib/muchos/ec2.py
@@ -33,7 +33,6 @@ class Ec2Cluster(ExistingCluster):
ExistingCluster.__init__(self, config)
def launch_node(self, hostname, services, sg_id):
-
request = self.init_request(hostname, services, sg_id)
request["MinCount"] = 1
@@ -116,7 +115,6 @@ class Ec2Cluster(ExistingCluster):
return group_id
def delete_security_group(self):
-
sg_id = None
ec2 = boto3.client("ec2")
try:
@@ -229,7 +227,7 @@ class Ec2Cluster(ExistingCluster):
sg_id = self.create_security_group()
instance_d = {}
- for (hostname, services) in list(self.config.nodes().items()):
+ for hostname, services in list(self.config.nodes().items()):
instance = self.launch_node(hostname, services, sg_id)
instance_d[instance["InstanceId"]] = hostname
diff --git a/lib/muchos/existing.py b/lib/muchos/existing.py
index cb5eaa6..d8a87ab 100644
--- a/lib/muchos/existing.py
+++ b/lib/muchos/existing.py
@@ -133,7 +133,7 @@ class ExistingCluster:
)
print("\n[zookeepers]", file=hosts_file)
- for (index, zk_host) in enumerate(
+ for index, zk_host in enumerate(
config.get_service_hostnames("zookeeper"), start=1
):
print("{0} id={1}".format(zk_host, index), file=hosts_file)
@@ -163,7 +163,7 @@ class ExistingCluster:
)
print("\n[nodes]", file=hosts_file)
- for (private_ip, hostname) in config.get_private_ip_hostnames():
+ for private_ip, hostname in config.get_private_ip_hostnames():
print(
"{0} ansible_ssh_host={1} node_type={2}".format(
hostname, private_ip, config.node_type(hostname)
@@ -177,13 +177,13 @@ class ExistingCluster:
self.add_specialized_configs(hosts_file)
print("\n[all:vars]", file=hosts_file)
- for (name, value) in sorted(host_vars.items()):
+ for name, value in sorted(host_vars.items()):
print("{0} = {1}".format(name, value), file=hosts_file)
with open(
path.join(config.deploy_path, "ansible/group_vars/all"), "w"
) as play_vars_file:
- for (name, value) in sorted(play_vars.items()):
+ for name, value in sorted(play_vars.items()):
print("{0}: {1}".format(name, value), file=play_vars_file)
# copy keys file to ansible/conf (if it exists)
diff --git a/lib/tests/azure/test_config.py b/lib/tests/azure/test_config.py
index 90e4e10..164a311 100644
--- a/lib/tests/azure/test_config.py
+++ b/lib/tests/azure/test_config.py
@@ -107,7 +107,7 @@ def test_azure_cluster():
assert c.proxy_public_ip() == "23.0.0.0"
assert c.proxy_private_ip() == "10.0.0.0"
assert c.get("general", "cluster_user") == (
- c.get("general", "cluster_group")
+ c.get("general", "cluster_group")
)
assert c.get_non_proxy() == [
("10.0.0.1", "leader2"),
diff --git a/lib/tests/ec2/test_config.py b/lib/tests/ec2/test_config.py
index df3d86c..02f8c72 100644
--- a/lib/tests/ec2/test_config.py
+++ b/lib/tests/ec2/test_config.py
@@ -111,7 +111,7 @@ def test_ec2_cluster():
assert c.proxy_public_ip() == "23.0.0.0"
assert c.proxy_private_ip() == "10.0.0.0"
assert c.get("general", "cluster_user") == (
- c.get("general", "cluster_group")
+ c.get("general", "cluster_group")
)
assert c.get_non_proxy() == [
("10.0.0.1", "leader2"),
diff --git a/scripts/cibuild b/scripts/cibuild
index be24894..4d0756a 100755
--- a/scripts/cibuild
+++ b/scripts/cibuild
@@ -6,7 +6,7 @@ nose2 -v -B
echo "SUCCESS: All nose2 tests completed."
echo "Running Ansible-lint..."
cd ..
-ANSIBLE_LIBRARY=./ansible/library ansible-lint -x
no-changed-when,risky-file-permissions,unnamed-task,var-naming,fqcn[action],fqcn[action-core],name[missing],name[casing],name[play],yaml[line-length],no-free-form,jinja[spacing],yaml[truthy]
ansible/ --exclude ansible/roles/azure/files/cloud-init.yml
+ansible-lint -x
no-changed-when,risky-file-permissions,unnamed-task,var-naming,fqcn[action],fqcn[action-core],name[missing],name[casing],name[play],yaml[line-length],no-free-form,jinja[spacing],yaml[truthy]
ansible/ --exclude ansible/roles/azure/files/cloud-init.yml
echo "SUCCESS: Ansible-lint completed."
echo "Running flake8 to check Python code..."
flake8
diff --git a/scripts/install-ansible-collections
b/scripts/install-ansible-collections
index 083f5ba..c43f2f0 100755
--- a/scripts/install-ansible-collections
+++ b/scripts/install-ansible-collections
@@ -1,2 +1,2 @@
#!/usr/bin/env bash
-ansible-galaxy collection install ansible.posix
+ansible-galaxy collection install ansible.posix==1.5.4
diff --git a/scripts/install-ansible-for-azure
b/scripts/install-ansible-for-azure
index a7388a8..799b5ad 100755
--- a/scripts/install-ansible-for-azure
+++ b/scripts/install-ansible-for-azure
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-ansible-galaxy collection install community.general:3.4.0
-ansible-galaxy collection install azure.azcollection:1.8.0
+ansible-galaxy collection install community.general==7.5.0
+ansible-galaxy collection install azure.azcollection==1.18.1
pip install -r
~/.ansible/collections/ansible_collections/azure/azcollection/requirements-azure.txt
-ansible-galaxy collection install community.azure:1.0.0
