This is an automated email from the ASF dual-hosted git repository. mykolabodnar pushed a commit to branch DLAB-1835 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit bd89d6f70f3afb26debfcc684d8ffef9c0fe6a0a Author: bodnarmykola <bodnarmyk...@gmail.com> AuthorDate: Sun Jun 7 22:24:33 2020 +0300 [DLAB-1835]- Private IP instead hostname for ssh connection changed --- .../src/general/conf/dlab.ini | 2 + .../general/scripts/aws/common_prepare_notebook.py | 5 +- .../src/general/scripts/aws/edge_configure.py | 18 ++++--- .../src/general/scripts/aws/jupyter_configure.py | 18 +++---- .../src/general/scripts/aws/project_prepare.py | 55 +++++++++++++--------- 5 files changed, 57 insertions(+), 41 deletions(-) diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini index 8ab5f9e..0aedd0b 100644 --- a/infrastructure-provisioning/src/general/conf/dlab.ini +++ b/infrastructure-provisioning/src/general/conf/dlab.ini @@ -94,6 +94,8 @@ private_subnet_prefix = 24 # subnet_id = ### Id of the subnet for notebooks and compute engines # subnet2_id = +### CIDR of the subnet for notebooks and compute engines +# private_subnet_cidr = ### Id of the vpc for whole dlab provisioning # vpc_id = ### Id of the secondary vpc for notebooks and compute engines diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py index 5c481ac..9a9bf51 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py +++ b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py @@ -112,7 +112,10 @@ if __name__ == "__main__": tag = {"Key": notebook_config['tag_name'], "Value": "{}-{}-{}-subnet".format(notebook_config['service_base_name'], notebook_config['project_name'], notebook_config['endpoint_name'])} - notebook_config['subnet_cidr'] = dlab.meta_lib.get_subnet_by_tag(tag) + if os.environ['aws_private_subnet_cidr']: + notebook_config['subnet_cidr'] = os.environ['aws_private_subnet_cidr'] + else: + notebook_config['subnet_cidr'] = dlab.meta_lib.get_subnet_by_tag(tag) keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']) with open('/root/result.json', 'w') as f: diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py b/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py index cfb4aa9..3a30bbd 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py +++ b/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py @@ -110,11 +110,9 @@ if __name__ == "__main__": for cidr in os.environ['conf_allowed_ip_cidr'].split(','): edge_conf['allowed_ip_cidr'].append(cidr.replace(' ', '')) - if edge_conf['network_type'] == 'public': - edge_conf['instance_hostname'] = dlab.meta_lib.get_instance_hostname(edge_conf['tag_name'], + edge_conf['instance_hostname'] = dlab.meta_lib.get_instance_hostname(edge_conf['tag_name'], edge_conf['instance_name']) - elif edge_conf['network_type'] == 'private': - edge_conf['instance_hostname'] = dlab.meta_lib.get_instance_ip_address( + edge_conf['instance_private_ip'] = dlab.meta_lib.get_instance_ip_address( edge_conf['tag_name'], edge_conf['instance_name']).get('Private') edge_conf['keyfile_name'] = "{}{}.pem".format(os.environ['conf_key_dir'], edge_conf['key_name']) @@ -142,7 +140,7 @@ if __name__ == "__main__": logging.info('[CREATING DLAB SSH USER]') print('[CREATING DLAB SSH USER]') params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format( - edge_conf['instance_hostname'], os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem", + edge_conf['instance_private_ip'], os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem", edge_conf['initial_user'], edge_conf['dlab_ssh_user'], edge_conf['sudo_group']) try: @@ -159,7 +157,7 @@ if __name__ == "__main__": print('[INSTALLING PREREQUISITES]') logging.info('[INSTALLING PREREQUISITES]') params = "--hostname {} --keyfile {} --user {} --region {}".\ - format(edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'], + format(edge_conf['instance_private_ip'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'], os.environ['aws_region']) try: local("~/scripts/{}.py {}".format('install_prerequisites', params)) @@ -184,7 +182,7 @@ if __name__ == "__main__": "vpc_cidrs": edge_conf['vpc_cidrs'], "allowed_ip_cidr": edge_conf['allowed_ip_cidr']} params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format( - edge_conf['instance_hostname'], edge_conf['keyfile_name'], json.dumps(additional_config), + edge_conf['instance_private_ip'], edge_conf['keyfile_name'], json.dumps(additional_config), edge_conf['dlab_ssh_user']) try: local("~/scripts/{}.py {}".format('configure_http_proxy', params)) @@ -204,7 +202,7 @@ if __name__ == "__main__": "user_keydir": os.environ['conf_key_dir'], "user_key": edge_conf['user_key']} params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format( - edge_conf['instance_hostname'], edge_conf['keyfile_name'], json.dumps(additional_config), + edge_conf['instance_private_ip'], edge_conf['keyfile_name'], json.dumps(additional_config), edge_conf['dlab_ssh_user']) try: local("~/scripts/{}.py {}".format('install_user_key', params)) @@ -222,7 +220,7 @@ if __name__ == "__main__": edge_conf['keycloak_client_secret'] = str(uuid.uuid4()) params = "--hostname {} --keyfile {} --user {} --keycloak_client_id {} --keycloak_client_secret {} " \ "--step_cert_sans '{}' ".format( - edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'], + edge_conf['instance_private_ip'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'], '{}-{}-{}'.format(edge_conf['service_base_name'], edge_conf['project_name'], edge_conf['endpoint_name']), edge_conf['keycloak_client_secret'], edge_conf['step_cert_sans']) @@ -237,7 +235,7 @@ if __name__ == "__main__": edge_conf['service_base_name'], os.environ['keycloak_auth_server_url'], os.environ['keycloak_realm_name'], os.environ['keycloak_user'], os.environ['keycloak_user_password'], edge_conf['keycloak_client_secret'], - edge_conf['instance_hostname'], edge_conf['instance_hostname'], edge_conf['project_name'], + edge_conf['instance_hostname'], edge_conf['instance_private_ip'], edge_conf['project_name'], edge_conf['endpoint_name']) try: local("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params)) diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py index cc53b22..bb48bcf 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py +++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py @@ -93,6 +93,8 @@ if __name__ == "__main__": edge_instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name) edge_instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private') + instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], + notebook_config['instance_name']).get('Private') notebook_config['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name) keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']) @@ -113,7 +115,7 @@ if __name__ == "__main__": logging.info('[CREATING DLAB SSH USER]') print('[CREATING DLAB SSH USER]') params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format( - instance_hostname, "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']), + instance_private_ip, "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']), notebook_config['initial_user'], notebook_config['dlab_ssh_user'], notebook_config['sudo_group']) try: @@ -132,7 +134,7 @@ if __name__ == "__main__": print('[CONFIGURE PROXY ON JUPYTER INSTANCE]') additional_config = {"proxy_host": edge_instance_hostname, "proxy_port": "3128"} params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}".format( - instance_hostname, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config), + instance_private_ip, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config), notebook_config['dlab_ssh_user']) try: local("~/scripts/{}.py {}".format('common_configure_proxy', params)) @@ -149,7 +151,7 @@ if __name__ == "__main__": logging.info('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]') print('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]') params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".format( - instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'], + instance_private_ip, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'], edge_instance_private_ip) try: local("~/scripts/{}.py {}".format('install_prerequisites', params)) @@ -176,7 +178,7 @@ if __name__ == "__main__": "--ip_address {8} " \ "--exploratory_name {9} " \ "--edge_ip {10}".\ - format(instance_hostname, + format(instance_private_ip, keyfile_name, os.environ['aws_region'], os.environ['notebook_spark_version'], @@ -203,7 +205,7 @@ if __name__ == "__main__": additional_config = {"user_keyname": notebook_config['user_keyname'], "user_keydir": os.environ['conf_key_dir']} params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format( - instance_hostname, keyfile_name, json.dumps(additional_config), notebook_config['dlab_ssh_user']) + instance_private_ip, keyfile_name, json.dumps(additional_config), notebook_config['dlab_ssh_user']) try: local("~/scripts/{}.py {}".format('install_user_key', params)) except: @@ -218,7 +220,7 @@ if __name__ == "__main__": print('[SETUP USER GIT CREDENTIALS]') logging.info('[SETUP USER GIT CREDENTIALS]') params = '--os_user {} --notebook_ip {} --keyfile "{}"' \ - .format(notebook_config['dlab_ssh_user'], instance_hostname, keyfile_name) + .format(notebook_config['dlab_ssh_user'], instance_private_ip, keyfile_name) try: local("~/scripts/{}.py {}".format('common_download_git_certfile', params)) local("~/scripts/{}.py {}".format('manage_git_creds', params)) @@ -235,7 +237,7 @@ if __name__ == "__main__": print('[POST CONFIGURING PROCESS') if notebook_config['notebook_image_name'] not in [notebook_config['expected_image_name'], 'None']: params = "--hostname {} --keyfile {} --os_user {} --nb_tag_name {} --nb_tag_value {}" \ - .format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], + .format(instance_private_ip, keyfile_name, notebook_config['dlab_ssh_user'], notebook_config['tag_name'], notebook_config['instance_name']) try: local("~/scripts/{}.py {}".format('common_remove_remote_kernels', params)) @@ -260,7 +262,7 @@ if __name__ == "__main__": "--type {} " \ "--exploratory_name {} " \ "--additional_info '{}'"\ - .format(edge_instance_hostname, + .format(edge_instance_private_ip, keyfile_name, notebook_config['dlab_ssh_user'], 'jupyter', diff --git a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py index 9d44ba5..0523fbc 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py @@ -184,34 +184,45 @@ if __name__ == "__main__": except KeyError: project_conf['vpc2_id'] = project_conf['vpc_id'] + tag = {"Key": project_conf['tag_name'], + "Value": "{0}-{1}-{2}-subnet".format(project_conf['service_base_name'], project_conf['project_name'], + project_conf['endpoint_name'])} + + if os.environ['aws_private_subnet_cidr']: + project_conf['private_subnet_cidr'] = os.environ['aws_private_subnet_cidr'] + else: + try: + try: + logging.info('[CREATE SUBNET]') + print('[CREATE SUBNET]') + params = "--vpc_id '{}' --infra_tag_name {} --infra_tag_value {} --prefix {} " \ + "--user_subnets_range '{}' --subnet_name {} --zone {}".format( + project_conf['vpc2_id'], project_conf['tag_name'], project_conf['service_base_name'], + project_conf['private_subnet_prefix'], os.environ['conf_user_subnets_range'], + project_conf['private_subnet_name'], + project_conf['zone']) + try: + local("~/scripts/{}.py {}".format('common_create_subnet', params)) + except: + traceback.print_exc() + raise Exception + except Exception as err: + dlab.fab.append_result("Failed to create subnet.", str(err)) + sys.exit(1) + project_conf['private_subnet_cidr'] = dlab.meta_lib.get_subnet_by_tag(tag) + except Exception as err: + dlab.fab.append_result("Failed to create subnet.", str(err)) + sys.exit(1) try: - logging.info('[CREATE SUBNET]') - print('[CREATE SUBNET]') - params = "--vpc_id '{}' --infra_tag_name {} --infra_tag_value {} --prefix {} " \ - "--user_subnets_range '{}' --subnet_name {} --zone {}".format( - project_conf['vpc2_id'], project_conf['tag_name'], project_conf['service_base_name'], - project_conf['private_subnet_prefix'], os.environ['conf_user_subnets_range'], - project_conf['private_subnet_name'], - project_conf['zone']) - try: - local("~/scripts/{}.py {}".format('common_create_subnet', params)) - except: - traceback.print_exc() - raise Exception + subnet_id = dlab.meta_lib.get_subnet_by_cidr(project_conf['private_subnet_cidr'], project_conf['vpc2_id']) + print('Subnet id: {}'.format(subnet_id)) + print('NEW SUBNET CIDR CREATED: {}'.format(project_conf['private_subnet_cidr'])) except Exception as err: - dlab.fab.append_result("Failed to create subnet.", str(err)) + dlab.fab.append_result("Failed to assign subnet.", str(err)) sys.exit(1) - tag = {"Key": project_conf['tag_name'], - "Value": "{0}-{1}-{2}-subnet".format(project_conf['service_base_name'], project_conf['project_name'], - project_conf['endpoint_name'])} - project_conf['private_subnet_cidr'] = dlab.meta_lib.get_subnet_by_tag(tag) - subnet_id = dlab.meta_lib.get_subnet_by_cidr(project_conf['private_subnet_cidr'], project_conf['vpc2_id']) - print('Subnet id: {}'.format(subnet_id)) - print('NEW SUBNET CIDR CREATED: {}'.format(project_conf['private_subnet_cidr'])) - try: logging.info('[CREATE EDGE ROLES]') print('[CREATE EDGE ROLES]') --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org For additional commands, e-mail: commits-h...@dlab.apache.org