This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2674 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c8288ad57a8a7ffe7a5eec330833eac5d94eeb9c Author: leonidfrolov <[email protected]> AuthorDate: Tue Feb 1 11:42:54 2022 +0200 [DATALAB-2674]: added os login for gcp --- infrastructure-provisioning/scripts/deploy_datalab.py | 2 ++ infrastructure-provisioning/src/general/conf/datalab.ini | 2 ++ .../src/general/lib/gcp/actions_lib.py | 6 +++++- .../src/general/scripts/gcp/common_create_instance.py | 3 ++- .../src/general/scripts/gcp/common_prepare_notebook.py | 7 +++++-- .../src/general/scripts/gcp/dataengine_prepare.py | 12 ++++++++---- .../src/general/scripts/gcp/project_prepare.py | 7 +++++-- .../src/general/scripts/gcp/ssn_prepare.py | 7 +++++-- 8 files changed, 34 insertions(+), 12 deletions(-) diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py index 9dd70e4..c039834 100644 --- a/infrastructure-provisioning/scripts/deploy_datalab.py +++ b/infrastructure-provisioning/scripts/deploy_datalab.py @@ -259,6 +259,8 @@ def build_parser(): help='One of more comma-separated GCP Firewall rules for SSN') gcp_parser.add_argument('--gcp_ssn_instance_size', type=str, default='n1-standard-2', help='The SSN instance shape') + gcp_parser.add_argument('--gcp_os_login_enabled', type=str, default='FALSE', + help='TRUE if os login enabled for gcp instances') gcp_required_args = gcp_parser.add_argument_group('Required arguments') gcp_required_args.add_argument('--gcp_region', type=str, required=True, help='GCP region') diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini index 26145fe..73ed7ba 100644 --- a/infrastructure-provisioning/src/general/conf/datalab.ini +++ b/infrastructure-provisioning/src/general/conf/datalab.ini @@ -228,6 +228,8 @@ datalake_enable = false # subnet_name = ### Names of the firewall rules for SSN # firewall_rules = +### True if os login is enabled for instances +# os_login_enabled = ### GCP region name for whole DataLab provisioning region = us-west1 ### GCP zone name for whole DataLab provisioning diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py index e4b070f..37e8d30 100644 --- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py @@ -320,7 +320,7 @@ class GCPActions: initial_user, image_name, secondary_image_name, service_account_name, instance_class, network_tag, labels, static_ip='', primary_disk_size='12', secondary_disk_size='30', - gpu_accelerator_type='None', gpu_accelerator_count='1'): + gpu_accelerator_type='None', gpu_accelerator_count='1', os_login_enabled='FALSE'): key = RSA.importKey(open(ssh_key_path, 'rb').read()) ssh_key = key.publickey().exportKey("OpenSSH").decode('UTF-8') unique_index = datalab.meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name) @@ -424,6 +424,10 @@ class GCPActions: { "key": "ssh-keys", "value": "{}:{}".format(initial_user, ssh_key) + }, + { + "key": "enable-oslogin", + "value": "{}".format(os_login_enabled) } ] }, diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py index d780b44..246675d 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py @@ -50,6 +50,7 @@ parser.add_argument('--gpu_accelerator_count', type=str, default='None') parser.add_argument('--network_tag', type=str, default='') parser.add_argument('--cluster_name', type=str, default='') parser.add_argument('--service_base_name', type=str, default='') +parser.add_argument('--os_login_enabled', type=str, default='FALSE') args = parser.parse_args() @@ -65,7 +66,7 @@ if __name__ == "__main__": args.secondary_image_name, args.service_account_name, args.instance_class, args.network_tag, json.loads(args.labels), args.static_ip, args.primary_disk_size, args.secondary_disk_size, args.gpu_accelerator_type, - args.gpu_accelerator_count) + args.gpu_accelerator_count, args.os_login_enabled) else: parser.print_help() sys.exit(2) diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py index dbacfab..9954592 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py @@ -148,6 +148,9 @@ if __name__ == "__main__": notebook_config['secondary_image_name'] = 'global/images/{}'.format( notebook_config['secondary_image_name'].get('name')) + if os.environ['gcp_os_login_enabled'] != 'FALSE': + notebook_config['gcp_os_login_enabled'] = 'TRUE' + notebook_config['gpu_accelerator_type'] = 'None' notebook_config['gpu_accelerator_count'] = 'None' @@ -193,7 +196,7 @@ if __name__ == "__main__": "--ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9} " \ "--secondary_image_name {10} --instance_class {11} --primary_disk_size {12} " \ "--secondary_disk_size {13} --gpu_accelerator_type {14} --gpu_accelerator_count {15} --network_tag {16} --labels '{17}' " \ - "--service_base_name {18}".\ + "--service_base_name {18} --os_login_enabled {19}".\ format(notebook_config['instance_name'], notebook_config['region'], notebook_config['zone'], notebook_config['vpc_name'], notebook_config['subnet_name'], notebook_config['instance_size'], notebook_config['ssh_key_path'], notebook_config['initial_user'], @@ -201,7 +204,7 @@ if __name__ == "__main__": notebook_config['secondary_image_name'], 'notebook', notebook_config['primary_disk_size'], notebook_config['secondary_disk_size'], notebook_config['gpu_accelerator_type'], notebook_config['gpu_accelerator_count'], notebook_config['network_tag'], - json.dumps(notebook_config['labels']), notebook_config['service_base_name']) + json.dumps(notebook_config['labels']), notebook_config['service_base_name'], notebook_config['gcp_os_login_enabled']) try: subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True) except: diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py index 84ee186..643354d 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py @@ -97,6 +97,10 @@ if __name__ == "__main__": if os.environ['conf_os_family'] == 'redhat': initial_user = 'ec2-user' sudo_group = 'wheel' + + if os.environ['gcp_os_login_enabled'] != 'FALSE': + data_engine['gcp_os_login_enabled'] = 'TRUE' + data_engine['cluster_name'] = "{}-{}-{}-de-{}".format(data_engine['service_base_name'], data_engine['project_name'], data_engine['endpoint_name'], @@ -189,14 +193,14 @@ if __name__ == "__main__": "--ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9} " \ "--secondary_image_name {10} --instance_class {11} --primary_disk_size {12} " \ "--secondary_disk_size {13} --gpu_accelerator_type {14} --gpu_accelerator_count {15} --network_tag {16} --cluster_name {17} " \ - "--labels '{18}' --service_base_name {19}". \ + "--labels '{18}' --service_base_name {19} --os_login_enabled {20}". \ format(data_engine['master_node_name'], data_engine['region'], data_engine['zone'], data_engine['vpc_name'], data_engine['subnet_name'], data_engine['master_size'], data_engine['ssh_key_path'], initial_user, data_engine['dataengine_service_account_name'], data_engine['primary_image_name'], data_engine['secondary_image_name'], 'dataengine', data_engine['primary_disk_size'], data_engine['secondary_disk_size'], data_engine['gpu_master_accelerator_type'], data_engine['gpu_master_accelerator_count'], data_engine['network_tag'], data_engine['cluster_name'], - json.dumps(data_engine['master_labels']), data_engine['service_base_name']) + json.dumps(data_engine['master_labels']), data_engine['service_base_name'], data_engine['gcp_os_login_enabled']) try: subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True) except: @@ -215,7 +219,7 @@ if __name__ == "__main__": "--instance_size {5} --ssh_key_path {6} --initial_user {7} --service_account_name {8} " \ "--image_name {9} --secondary_image_name {10} --instance_class {11} --primary_disk_size {12} " \ "--secondary_disk_size {13} --gpu_accelerator_type {14} --gpu_accelerator_count {15} --network_tag {16} --cluster_name {17} " \ - "--labels '{18}' --service_base_name {19}". \ + "--labels '{18}' --service_base_name {19} --os_login_enabled {20}". \ format(slave_name, data_engine['region'], data_engine['zone'], data_engine['vpc_name'], data_engine['subnet_name'], data_engine['slave_size'], data_engine['ssh_key_path'], initial_user, data_engine['dataengine_service_account_name'], @@ -224,7 +228,7 @@ if __name__ == "__main__": data_engine['secondary_disk_size'], data_engine['gpu_slave_accelerator_type'], data_engine['gpu_slave_accelerator_count'], data_engine['network_tag'], data_engine['cluster_name'], json.dumps(data_engine['slave_labels']), - data_engine['service_base_name']) + data_engine['service_base_name'], data_engine['gcp_os_login_enabled']) try: subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True) except: diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py index d5a06c9..02e9667 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py @@ -500,18 +500,21 @@ if __name__ == "__main__": project_conf['initial_user'] = 'ec2-user' project_conf['sudo_group'] = 'wheel' + if os.environ['gcp_os_login_enabled'] != 'FALSE': + project_conf['gcp_os_login_enabled'] = 'TRUE' + try: project_conf['static_ip'] = \ GCPMeta.get_static_address(project_conf['region'], project_conf['static_address_name'])['address'] logging.info('[CREATE EDGE INSTANCE]') params = "--instance_name {} --region {} --zone {} --vpc_name {} --subnet_name {} --instance_size {} " \ "--ssh_key_path {} --initial_user {} --service_account_name {} --image_name {} --instance_class {} " \ - "--static_ip {} --network_tag {} --labels '{}' --service_base_name {}".format( + "--static_ip {} --network_tag {} --labels '{}' --service_base_name {} --os_login_enabled {}".format( project_conf['instance_name'], project_conf['region'], project_conf['zone'], project_conf['vpc_name'], project_conf['subnet_name'], project_conf['instance_size'], project_conf['ssh_key_path'], project_conf['initial_user'], project_conf['edge_service_account_name'], project_conf['image_name'], 'edge', project_conf['static_ip'], project_conf['network_tag'], - json.dumps(project_conf['instance_labels']), project_conf['service_base_name']) + json.dumps(project_conf['instance_labels']), project_conf['service_base_name'], project_conf['gcp_os_login_enabled']) try: subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True) except: diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py index 59f0658..9346d15 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py @@ -251,6 +251,9 @@ if __name__ == "__main__": GCPActions.remove_vpc(ssn_conf['vpc_name']) sys.exit(1) + if os.environ['gcp_os_login_enabled'] != 'FALSE': + ssn_conf['gcp_os_login_enabled'] = 'TRUE' + if os.environ['conf_os_family'] == 'debian': ssn_conf['initial_user'] = 'ubuntu' ssn_conf['sudo_group'] = 'sudo' @@ -265,12 +268,12 @@ if __name__ == "__main__": params = "--instance_name {0} --region {1} --zone {2} --vpc_name {3} --subnet_name {4} --instance_size {5}"\ " --ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9}"\ " --instance_class {10} --static_ip {11} --network_tag {12} --labels '{13}' " \ - "--primary_disk_size {14} --service_base_name {15}".\ + "--primary_disk_size {14} --service_base_name {15} --os_login_enabled {16}".\ format(ssn_conf['instance_name'], ssn_conf['region'], ssn_conf['zone'], ssn_conf['vpc_name'], ssn_conf['subnet_name'], ssn_conf['instance_size'], ssn_conf['ssh_key_path'], ssn_conf['initial_user'], ssn_conf['service_account_name'], ssn_conf['image_name'], 'ssn', ssn_conf['static_ip'], ssn_conf['network_tag'], json.dumps(ssn_conf['instance_labels']), '20', - ssn_conf['service_base_name']) + ssn_conf['service_base_name'], ssn_conf['gcp_os_login_enabled']) try: subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True) except: --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
