This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2674 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 5122ddefe0ba3ca94a9dfcd27915f8c40b50a1c0 Author: leonidfrolov <[email protected]> AuthorDate: Mon Feb 7 15:49:53 2022 +0200 [DATALAB-2674]: cmek argument for gcp buckets --- infrastructure-provisioning/scripts/deploy_datalab.py | 3 +++ infrastructure-provisioning/src/general/conf/datalab.ini | 2 ++ infrastructure-provisioning/src/general/lib/gcp/actions_lib.py | 6 ++++-- .../src/general/scripts/gcp/common_create_bucket.py | 3 ++- .../src/general/scripts/gcp/project_prepare.py | 8 ++++++++ 5 files changed, 19 insertions(+), 3 deletions(-) diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py index 67db8da..f4587a8 100644 --- a/infrastructure-provisioning/scripts/deploy_datalab.py +++ b/infrastructure-provisioning/scripts/deploy_datalab.py @@ -265,6 +265,9 @@ def build_parser(): help='"TRUE" to block project ssh keys for gcp instances') gcp_parser.add_argument('--gcp_bucket_enable_versioning', type=str, default='false', help='"true" to enable versioning for gcp storage buckets') + gcp_parser.add_argument('--gcp_cmek_resource_name', type=str, default='', + help='customer managed encryption key resource name ' + 'e.g. projects/{project_name}/locations/{us}/keyRings/{keyring_name}/cryptoKeys/{key_name}') gcp_required_args = gcp_parser.add_argument_group('Required arguments') gcp_required_args.add_argument('--gcp_region', type=str, required=True, help='GCP region') diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini index c5bac75..b495302 100644 --- a/infrastructure-provisioning/src/general/conf/datalab.ini +++ b/infrastructure-provisioning/src/general/conf/datalab.ini @@ -234,6 +234,8 @@ os_login_enabled = FALSE block_project_ssh_keys = FALSE ### True if versioning is enabled for buckets bucket_enable_versioning = false +### gcp customer managed encryption key to use +# cmek_resource_name = ### GCP region name for whole DataLab provisioning region = us-west1 ### GCP zone name for whole DataLab provisioning diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py index e92d835..531d6fc 100644 --- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py @@ -210,14 +210,16 @@ class GCPActions: traceback.print_exc(file=sys.stdout) - def create_bucket(self, bucket_name, versioning_enabled='false'): + def create_bucket(self, bucket_name, versioning_enabled='false', cmek_resource_name=''): try: bucket_params = { "name": bucket_name, "versioning": { "enabled": "{}".format(versioning_enabled) - } + } } + if cmek_resource_name != '': + bucket_params["encryption"] = {"defaultKmsKeyName": cmek_resource_name} bucket = self.storage_client.create_bucket(project=self.project, body=bucket_params) print('Bucket {} created.'.format(bucket.name)) except Exception as err: diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py index 0c63b2d..0291dbb 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py @@ -32,6 +32,7 @@ parser = argparse.ArgumentParser() parser.add_argument('--bucket_name', type=str, default='') parser.add_argument('--tags', type=str, default='') parser.add_argument('--versioning_enabled', type=str, default='false') +parser.add_argument('--cmek_resource_name', type=str, default='') args = parser.parse_args() if __name__ == "__main__": @@ -40,7 +41,7 @@ if __name__ == "__main__": logging.info("REQUESTED BUCKET {} ALREADY EXISTS".format(args.bucket_name)) else: logging.info("Creating Bucket {}".format(args.bucket_name)) - GCPActions().create_bucket(args.bucket_name, args.versioning_enabled) + GCPActions().create_bucket(args.bucket_name, args.versioning_enabled, args.cmek_resource_name) GCPActions().add_bucket_labels(args.bucket_name, json.loads(args.tags)) else: parser.print_help() diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py index 1ebbcae..56591cf 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py @@ -122,6 +122,7 @@ if __name__ == "__main__": project_conf['user_subnets_range'] = '' project_conf['gcp_bucket_enable_versioning'] = os.environ['gcp_bucket_enable_versioning'] + project_conf['gcp_cmek_resource_name'] = os.environ['gcp_cmek_resource_name'] # FUSE in case of absence of user's key try: project_conf['user_key'] = os.environ['key'] @@ -403,6 +404,10 @@ if __name__ == "__main__": params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['shared_bucket_name'], json.dumps(project_conf['shared_bucket_tags']), project_conf['gcp_bucket_enable_versioning']) + + if project_conf['gcp_cmek_resource_name'] != '': + params = '{} --cmek_resource_name {}'.format(params, project_conf['gcp_cmek_resource_name']) + try: subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True) except: @@ -420,6 +425,9 @@ if __name__ == "__main__": json.dumps(project_conf['bucket_tags']), project_conf['gcp_bucket_enable_versioning']) + if project_conf['gcp_cmek_resource_name'] != '': + params = '{} --cmek_resource_name {}'.format(params, project_conf['gcp_cmek_resource_name']) + try: subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True) except: --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
