This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-1408 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 14f36c05fd0ce5f2c92589329649f382f41f399c Author: leonidfrolov <[email protected]> AuthorDate: Tue Aug 23 16:01:19 2022 +0300 [DATALAB-1408]: added some creation changes --- .../scripts/azure/dataengine-service_prepare.py | 107 ++++++++++++++++++++- .../scripts/gcp/dataengine-service_create.py | 2 +- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py index 177275f78..7bd369910 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py @@ -1,5 +1,26 @@ #!/usr/bin/python3 +# ***************************************************************************** +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ****************************************************************************** + import datalab.actions_lib import datalab.fab import datalab.meta_lib @@ -42,7 +63,7 @@ CONTAINER_NAME = 'hdinsight' # Blob Storage endpoint suffix. BLOB_ENDPOINT_SUFFIX = '.blob.core.windows.net' -def create_cluster_parameters(): +def create_cluster_parameters(LOCATION, CLUSTER_LOGIN_USER_NAME, PASSWORD, SSH_USER_NAME): # Returns cluster parameters @@ -92,7 +113,7 @@ def create_cluster_parameters(): storage_profile=StorageProfile( storageaccounts=[ StorageAccount( - name=STORAGE_ACCOUNT_NAME + BLOB_ENDPOINT_SUFFIX, + name=STORAGE_ACCOUNT_NAME + ".blob.core.windows.net", key=STORAGE_ACCOUNT_KEY, container=CONTAINER_NAME.lower(), is_default=True @@ -104,4 +125,84 @@ def create_cluster_parameters(): if __name__ == "__main__": #params = create_cluster_parameters() - #create_hdinsight_cluster(RESOURCE_GROUP_NAME,CLUSTER_NAME, params) \ No newline at end of file + #create_hdinsight_cluster(RESOURCE_GROUP_NAME,CLUSTER_NAME, params) + + try: + AzureMeta = datalab.meta_lib.AzureMeta() + AzureActions = datalab.actions_lib.AzureActions() + logging.info('Generating infrastructure names and tags') + hdinsight_conf = dict() + if 'exploratory_name' in os.environ: + hdinsight_conf['exploratory_name'] = os.environ['exploratory_name'].replace('_', '-').lower() + else: + hdinsight_conf['exploratory_name'] = '' + if 'computational_name' in os.environ: + hdinsight_conf['computational_name'] = os.environ['computational_name'].replace('_', '-').lower() + else: + hdinsight_conf['computational_name'] = '' + + hdinsight_conf['service_base_name'] = (os.environ['conf_service_base_name']) + hdinsight_conf['edge_user_name'] = (os.environ['edge_user_name']) + hdinsight_conf['project_name'] = (os.environ['project_name']).replace('_', '-').lower() + hdinsight_conf['project_tag'] = hdinsight_conf['project_name'] + hdinsight_conf['endpoint_name'] = (os.environ['endpoint_name']).replace('_', '-').lower() + hdinsight_conf['endpoint_tag'] = hdinsight_conf['endpoint_name'] + hdinsight_conf['key_name'] = os.environ['conf_key_name'] + hdinsight_conf['key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) + hdinsight_conf['zone'] = os.environ['gcp_zone'] + hdinsight_conf['resource_group_name'] = os.environ['azure_resource_group_name'] + hdinsight_conf['region'] = os.environ['azure_region'] + data_engine['vpc_name'] = os.environ['azure_vpc_name'] + data_engine['private_subnet_name'] = '{}-{}-{}-subnet'.format(data_engine['service_base_name'], + data_engine['project_name'], + data_engine['endpoint_name']) + data_engine['private_subnet_cidr'] = AzureMeta.get_subnet(data_engine['resource_group_name'], + data_engine['vpc_name'], + data_engine['private_subnet_name']).address_prefix + data_engine['cluster_name'] = '{}-{}-{}-des-{}'.format(data_engine['service_base_name'], + data_engine['project_name'], + data_engine['endpoint_name'], + data_engine['computational_name']) + + + + hdinsight_conf['subnet'] = '{0}-{1}-{2}-subnet'.format(hdinsight_conf['service_base_name'], + hdinsight_conf['project_name'], + hdinsight_conf['endpoint_name']) + hdinsight_conf['cluster_name'] = '{0}-{1}-{2}-des-{3}'.format(hdinsight_conf['service_base_name'], + hdinsight_conf['project_name'], + hdinsight_conf['endpoint_name'], + hdinsight_conf['computational_name']) + hdinsight_conf['cluster_tag'] = '{0}-{1}-{2}-ps'.format(hdinsight_conf['service_base_name'], + hdinsight_conf['project_name'], + hdinsight_conf['endpoint_name']) + hdinsight_conf['bucket_name'] = '{0}-{1}-{2}-bucket'.format(hdinsight_conf['service_base_name'], + hdinsight_conf['project_name'], + hdinsight_conf['endpoint_name']) + + hdinsight_conf['edge_instance_hostname'] = '{0}-{1}-{2}-edge'.format(hdinsight_conf['service_base_name'], + hdinsight_conf['project_name'], + hdinsight_conf['endpoint_name']) + hdinsight_conf['datalab_ssh_user'] = os.environ['conf_os_user'] + except Exception as err: + datalab.fab.append_result("Failed to generate variables dictionary. Exception:" + str(err)) + sys.exit(1) + + try: + logging.info('[Creating HDInsight Cluster]') + params = "--region {0} --bucket {1} --params '{2}'".format(hdinsight_conf['region'], + hdinsight_conf['bucket_name'], + json.dumps(hdinsight_cluster)) + + try: + subprocess.run("~/scripts/{}.py {}".format('dataengine-service_create', params), shell=True, check=True) + except: + traceback.print_exc() + raise Exception + + keyfile_name = "/root/keys/{}.pem".format(hdinsight_conf['key_name']) + subprocess.run('rm /response/.hdinsight_creating_{}'.format(os.environ['exploratory_name']), shell=True, check=True) + except Exception as err: + datalab.fab.append_result("Failed to create hdinsight Cluster.", str(err)) + subprocess.run('rm /response/.hdinsight_creating_{}'.format(os.environ['exploratory_name']), shell=True, check=True) + sys.exit(1) diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py index 978d2d4fe..2ea506d7b 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py @@ -81,4 +81,4 @@ if __name__ == "__main__": build_dataproc_cluster(args, cluster_name) send_parser_job(args, cluster_name, cluster_version) - sys.exit(0) \ No newline at end of file + sys.exit(0) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
