This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-1408 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f153bbeb365ae70bead702f44b0aa153b41c85c6 Author: leonidfrolov <[email protected]> AuthorDate: Tue Sep 6 14:30:14 2022 +0300 [DATALAB-1408]: changed container type --- .../src/general/lib/azure/actions_lib.py | 25 +++++++++++----------- .../scripts/azure/common_create_storage_account.py | 7 +++--- .../scripts/azure/dataengine-service_prepare.py | 16 ++++++-------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py index 563095bf4..db9088aa6 100644 --- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py @@ -450,25 +450,23 @@ class AzureActions: file=sys.stdout)})) traceback.print_exc(file=sys.stdout) - def create_storage_account(self, resource_group_name, account_name, region, tags): + def create_storage_account(self, resource_group_name, account_name, region, tags, kind='BlobStorage'): try: ssn_network_id = datalab.meta_lib.AzureMeta().get_subnet(resource_group_name, vpc_name=os.environ['azure_vpc_name'], - subnet_name=os.environ['azure_subnet_name'] - ).id + subnet_name=os.environ['azure_subnet_name']).id edge_network_id = datalab.meta_lib.AzureMeta().get_subnet(resource_group_name, - vpc_name=os.environ['azure_vpc_name'], - subnet_name='{}-{}-{}-subnet'.format( - os.environ['conf_service_base_name'], - (os.environ['project_name']), - (os.environ['endpoint_name'])) - ).id + vpc_name=os.environ['azure_vpc_name'], + subnet_name='{}-{}-{}-subnet'.format( + os.environ['conf_service_base_name'], + (os.environ['project_name']), + (os.environ['endpoint_name']))).id result = self.storage_client.storage_accounts.begin_create( resource_group_name, account_name, { "sku": {"name": "Standard_LRS"}, - "kind": "BlobStorage", + "kind": kind, "location": region, "tags": tags, "access_tier": "Hot", @@ -520,13 +518,14 @@ class AzureActions: file=sys.stdout)})) traceback.print_exc(file=sys.stdout) - def create_blob_container(self, resource_group_name, account_name, container_name): + def create_blob_container(self, account_name, container_name): try: - block_blob_service = BlobServiceClient(account_url="https://"; + account_name + ".blob.core.windows.net/", credential=self.credential) + block_blob_service = BlobServiceClient(account_url="https://"; + account_name + ".blob.core.windows.net/", + credential=self.credential) result = block_blob_service.create_container( container_name, { - "public_access": "Off" + "public_access": "Off" } ) return result diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py index 17676b084..8c94592da 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py +++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py @@ -34,6 +34,7 @@ parser.add_argument('--container_name', type=str, default='') parser.add_argument('--account_tags', type=str, default='{"empty":"string"}') parser.add_argument('--resource_group_name', type=str, default='') parser.add_argument('--region', type=str, default='') +parser.add_argument('--storage_account_kind', type=str, default='BlobStorage') args = parser.parse_args() if __name__ == "__main__": @@ -50,9 +51,9 @@ if __name__ == "__main__": if check.name_available: logging.info("Creating storage account {}.".format(account_name)) storage_account = AzureActions().create_storage_account(args.resource_group_name, account_name, - args.region, account_tags) - blob_container = AzureActions().create_blob_container(args.resource_group_name, account_name, - args.container_name) + args.region, account_tags, + args.storage_account_kind) + blob_container = AzureActions().create_blob_container(account_name, args.container_name) logging.info("STORAGE ACCOUNT {} has been created".format(account_name)) logging.info("CONTAINER {} has been created".format(args.container_name)) else: diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py index 5725c2c0c..49f636566 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py @@ -77,14 +77,8 @@ if __name__ == "__main__": hdinsight_conf['release_label'] = os.environ['hdinsight_version'] key = RSA.importKey(open(hdinsight_conf['key_path'], 'rb').read()) ssh_admin_pubkey = key.publickey().exportKey("OpenSSH").decode('UTF-8') - hdinsight_conf['container_name'] = ('{0}-bucket'.format(hdinsight_conf['service_base_name'], - hdinsight_conf['project_name'], - hdinsight_conf['endpoint_name'], - hdinsight_conf['cluster_name'])).lower() - hdinsight_conf['storage_account_name_tag'] = ('{0}-bucket'.format(hdinsight_conf['service_base_name'], - hdinsight_conf['project_name'], - hdinsight_conf['endpoint_name'], - hdinsight_conf['cluster_name'])).lower() + hdinsight_conf['container_name'] = ('{}-bucket'.format(hdinsight_conf['cluster_name'])).lower() + hdinsight_conf['storage_account_name_tag'] = ('{}-bucket'.format(hdinsight_conf['cluster_name'])).lower() hdinsight_conf['storage_account_tags'] = {"Name": hdinsight_conf['storage_account_name_tag'], "SBN": hdinsight_conf['service_base_name'], "project_tag": hdinsight_conf['project_name'], @@ -112,7 +106,8 @@ if __name__ == "__main__": try: logging.info('[CREATE STORAGE ACCOUNT AND CONTAINERS]') - params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \ + params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {} " \ + "--storage_account_kind StorageV2". \ format(hdinsight_conf['container_name'], json.dumps(hdinsight_conf['storage_account_tags']), hdinsight_conf['resource_group_name'], hdinsight_conf['region']) try: @@ -156,5 +151,8 @@ if __name__ == "__main__": except Exception as err: datalab.fab.append_result("Failed to create hdinsight Cluster.", str(err)) + for storage_account in AzureMeta.list_storage_accounts(hdinsight_conf['resource_group_name']): + if hdinsight_conf['storage_account_name_tag'] == storage_account.tags["Name"]: + AzureActions.remove_storage_account(hdinsight_conf['resource_group_name'], storage_account.name) #subprocess.run('rm /response/.hdinsight_creating_{}'.format(os.environ['exploratory_name']), shell=True, check=True) sys.exit(1) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
