This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-1408 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 49f5dbbc1b336d4c49ed93153aac05ca97338b9d Author: leonidfrolov <[email protected]> AuthorDate: Tue Sep 6 12:20:25 2022 +0300 [DATALAB-1408]: added vpc and edge subnet usage --- .../scripts/azure/dataengine-service_create.py | 17 ++++++++++++++--- .../scripts/azure/dataengine-service_prepare.py | 19 +++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py index fe93fc030..d31b001a1 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py +++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py @@ -46,6 +46,8 @@ parser.add_argument('--storage_account_key', type=str, help='') parser.add_argument('--container_name', type=str, help='') parser.add_argument('--tags', type=str, help='') parser.add_argument('--public_key', type=str, help='') +parser.add_argument('--vpc_id', type=str, help='') +parser.add_argument('--subnet', type=str, help='') args = parser.parse_args() @@ -56,7 +58,7 @@ def build_hdinsight_cluster(resource_group_name, cluster_name, params): def create_cluster_parameters(location, tags, cluster_version, cluster_login_username, password, master_instance_type, worker_count, worker_instance_type, storage_account_name, storage_account_key, - container_name, public_key): + container_name, public_key, vpc_id, subnet): # Returns cluster parameters @@ -92,6 +94,10 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use ] } ) + ), + virtual_network_profile=VirtualNetworkProfile( + id=vpc_id, + subnet=subnet ) ), Role( @@ -107,6 +113,10 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use ] } ) + ), + virtual_network_profile=VirtualNetworkProfile( + id=vpc_id, + subnet=subnet ) ) ] @@ -129,12 +139,13 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use ############## if __name__ == "__main__": - parser.print_help() + #parser.print_help() password = '' params = create_cluster_parameters(args.location, json.loads(args.tags), args.cluster_version, 'datalab-user', password, args.master_instance_type, args.worker_count, args.worker_instance_type, args.storage_account_name, args.storage_account_key, - args.container_name, args.public_key) + args.container_name, args.public_key, args.vpc_id, args.subnet) + build_hdinsight_cluster(args.resource_group_name, args.cluster_name, params) logfile = '{}_creation.log'.format(args.cluster_name) diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py index d69929508..5725c2c0c 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py @@ -91,6 +91,20 @@ if __name__ == "__main__": "endpoint_tag": hdinsight_conf['endpoint_name'], os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'], hdinsight_conf['tag_name']: hdinsight_conf['storage_account_name_tag']} + + hdinsight_conf['vpc_name'] = os.environ['azure_vpc_name'] + + hdinsight_conf['vpc_id'] = AzureMeta.get_vpc(hdinsight_conf['resource_group_name'], + hdinsight_conf['vpc_name']).id + + hdinsight_conf['subnet_name'] = '{}-{}-{}-subnet'.format(hdinsight_conf['service_base_name'], + hdinsight_conf['project_name'], + hdinsight_conf['endpoint_name']) + + hdinsight_conf['edge_network_id'] = AzureMeta.get_subnet(hdinsight_conf['resource_group_name'], + hdinsight_conf['vpc_name'], + hdinsight_conf['subnet_name']).id + except Exception as err: datalab.fab.append_result("Failed to generate variables dictionary. Exception:" + str(err)) sys.exit(1) @@ -125,13 +139,14 @@ if __name__ == "__main__": "--master_instance_type {} --worker_instance_type {} " \ "--worker_count {} --storage_account_name {} " \ "--storage_account_key '{}' --container_name {} " \ - "--tags '{}' --public_key '{}'"\ + "--tags '{}' --public_key '{}' --vpc_id {} --subnet {}"\ .format(hdinsight_conf['resource_group_name'], hdinsight_conf['cluster_name'], hdinsight_conf['release_label'], hdinsight_conf['region'], os.environ['hdinsight_master_instance_type'], os.environ['hdinsight_slave_instance_type'], hdinsight_conf['hdinsight_worker_count'], hdinsight_conf['storage_account_name'], hdinsight_conf['storage_account_key'], hdinsight_conf['container_name'], - json.dumps(hdinsight_conf['cluster_tags']), ssh_admin_pubkey) + json.dumps(hdinsight_conf['cluster_tags']), ssh_admin_pubkey, hdinsight_conf['vpc_id'], + hdinsight_conf['edge_network_id']) try: subprocess.run("~/scripts/{}.py {}".format('dataengine-service_create', params), shell=True, check=True) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
