This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2409 by this push:
new 9b6b3a7 [DATALAB-2409] - [AWS] SSN configuration script refactored
9b6b3a7 is described below
commit 9b6b3a7c725ce02bea1184d79e49e23ac4409ca8
Author: bodnarmykola <[email protected]>
AuthorDate: Mon Aug 2 15:50:06 2021 +0300
[DATALAB-2409] - [AWS] SSN configuration script refactored
---
.../src/general/lib/os/fab.py | 20 ++++----
.../src/general/scripts/aws/ssn_prepare.py | 56 +++++++++++-----------
2 files changed, 38 insertions(+), 38 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py
b/infrastructure-provisioning/src/general/lib/os/fab.py
index fc9c570..68f9c3a 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -38,6 +38,16 @@ from fabric import *
from patchwork.files import exists
from patchwork import files
+local_log_filename = "{}_{}.log".format(os.environ['conf_resource'],
os.environ['request_id'])
+local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" +
local_log_filename
+logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename='{}'.format(local_log_filepath),
+ filemode='w')
+console = logging.StreamHandler()
+console.setLevel(logging.INFO)
+logging.getLogger('').addHandler(console)
+
def ensure_python_venv(python_venv_version):
try:
if not exists(conn,
'/opt/python/python{}'.format(python_venv_version)):
@@ -1152,13 +1162,3 @@ def init_datalab_connection(hostname, username, keyfile):
except:
traceback.print_exc()
sys.exit(1)
-
-def logging_datalab():
- local_log_filename = "{}_{}.log".format(os.environ['conf_resource'],
os.environ['request_id'])
- local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" +
local_log_filename
- logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
- level=logging.DEBUG,
- filename='{}'.format(local_log_filepath),
- filemode='w')
- console = logging.StreamHandler()
- logging.getLogger('').addHandler(console)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
index 557555f..1a10b7b 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
@@ -43,20 +43,20 @@ def cleanup_aws_resources(tag_name, service_base_name):
if __name__ == "__main__":
#creating aws config file
try:
- datalab.fab.logging_datalab.info('[CREATE AWS CONFIG FILE]')
+ logging.info('[CREATE AWS CONFIG FILE]')
if 'aws_access_key' in os.environ and 'aws_secret_access_key' in
os.environ:
datalab.actions_lib.create_aws_config_files(generate_full_config=True)
else:
datalab.actions_lib.create_aws_config_files()
except Exception as err:
- datalab.fab.logging_datalab.info('Unable to create configuration')
+ logging.info('Unable to create configuration')
datalab.fab.append_result("Unable to create configuration", err)
traceback.print_exc()
sys.exit(1)
#deriving variables for ssn node deployment
try:
- datalab.fab.logging_datalab.info('[DERIVING NAMES]')
+ logging.info('[DERIVING NAMES]')
ssn_conf = dict()
ssn_conf['service_base_name'] = os.environ['conf_service_base_name'] =
datalab.fab.replace_multi_symbols(
os.environ['conf_service_base_name'][:20], '-', True)
@@ -94,11 +94,11 @@ if __name__ == "__main__":
#checking sbn for unique value
try:
if datalab.meta_lib.get_instance_by_name(ssn_conf['tag_name'],
ssn_conf['instance_name']):
- datalab.fab.logging_datalab.info("Service base name should be
unique and less or equal 20 symbols. Please try again.")
+ logging.info("Service base name should be unique and less or equal
20 symbols. Please try again.")
sys.exit(1)
except Exception as err:
datalab.fab.append_result("Failed to make predeployment check.",
str(err))
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
traceback.print_exc()
sys.exit(1)
@@ -107,7 +107,7 @@ if __name__ == "__main__":
if 'aws_vpc_id' in os.environ and os.environ['aws_vpc_id'] != '':
ssn_conf['aws_vpc_id'] = os.environ['aws_vpc_id']
else:
- datalab.fab.logging_datalab.info('[CREATE VPC AND ROUTE TABLE]')
+ logging.info('[CREATE VPC AND ROUTE TABLE]')
params = "--vpc {} --region {} --infra_tag_name {}
--infra_tag_value {} --vpc_name {}".format(
ssn_conf['vpc_cidr'], ssn_conf['region'],
ssn_conf['tag_name'], ssn_conf['service_base_name'],
ssn_conf['vpc_name'])
@@ -121,7 +121,7 @@ if __name__ == "__main__":
for cidr in
datalab.meta_lib.get_vpc_cidr_by_id(ssn_conf['aws_vpc_id']):
ssn_conf['allowed_vpc_cidr_ip_ranges'].append({"CidrIp": cidr})
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create VPC", str(err))
sys.exit(1)
@@ -131,7 +131,7 @@ if __name__ == "__main__":
and os.environ['aws_vpc2_id'] != '':
ssn_conf['aws_vpc2_id'] = os.environ['aws_vpc2_id']
elif os.environ['conf_duo_vpc_enable'] == 'true':
- datalab.fab.logging_datalab.info('[CREATE SECONDARY VPC AND ROUTE
TABLE]')
+ logging.info('[CREATE SECONDARY VPC AND ROUTE TABLE]')
params = "--vpc {} --region {} --infra_tag_name {}
--infra_tag_value {} --secondary" \
" --vpc_name {}".format(ssn_conf['vpc2_cidr'],
ssn_conf['region'], ssn_conf['tag2_name'],
ssn_conf['service_base_name'],
ssn_conf['vpc2_name'])
@@ -143,7 +143,7 @@ if __name__ == "__main__":
ssn_conf['aws_vpc2_id'] =
datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag2_name'],
ssn_conf['service_base_name'])
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create secondary VPC", str(err))
cleanup_aws_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
@@ -153,7 +153,7 @@ if __name__ == "__main__":
if 'aws_subnet_id' in os.environ and os.environ['aws_subnet_id'] != '':
ssn_conf['aws_subnet_id'] = os.environ['aws_subnet_id']
else:
- datalab.fab.logging_datalab.info('[CREATE SUBNET]')
+ logging.info('[CREATE SUBNET]')
params = "--vpc_id {0} --username {1} --infra_tag_name {2}
--infra_tag_value {3} --prefix {4} " \
"--ssn {5} --zone {6} --subnet_name
{7}".format(ssn_conf['aws_vpc_id'], 'ssn',
ssn_conf['tag_name'],
@@ -169,7 +169,7 @@ if __name__ == "__main__":
ssn_conf['aws_vpc_id'])
datalab.actions_lib.enable_auto_assign_ip(ssn_conf['aws_subnet_id'])
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create Subnet", str(err))
cleanup_aws_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
@@ -177,15 +177,15 @@ if __name__ == "__main__":
#creating peering connection
try:
if os.environ['conf_duo_vpc_enable'] == 'true' and
ssn_conf['aws_vpc_id'] and ssn_conf['aws_vpc2_id']:
- datalab.fab.logging_datalab.info('[CREATE PEERING CONNECTION]')
+ logging.info('[CREATE PEERING CONNECTION]')
ssn_conf['aws_peering_id'] =
datalab.actions_lib.create_peering_connection(
ssn_conf['aws_vpc_id'], ssn_conf['aws_vpc2_id'],
ssn_conf['service_base_name'])
- datalab.fab.logging_datalab.info('PEERING CONNECTION ID:' +
ssn_conf['aws_peering_id'])
+ logging.info('PEERING CONNECTION ID:' + ssn_conf['aws_peering_id'])
datalab.actions_lib.create_route_by_id(ssn_conf['aws_subnet_id'],
ssn_conf['aws_vpc_id'],
ssn_conf['aws_peering_id'],
datalab.meta_lib.get_cidr_by_vpc(ssn_conf['aws_vpc2_id']))
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create peering connection",
str(err))
cleanup_aws_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
@@ -195,7 +195,7 @@ if __name__ == "__main__":
if 'aws_security_groups_ids' in os.environ and
os.environ['aws_security_groups_ids'] != '':
ssn_conf['aws_security_groups_ids'] =
os.environ['aws_security_groups_ids']
else:
- datalab.fab.logging_datalab.info('[CREATE SG FOR SSN]')
+ logging.info('[CREATE SG FOR SSN]')
ssn_conf['ingress_sg_rules_template'] =
datalab.meta_lib.format_sg([
{
"PrefixListIds": [],
@@ -250,14 +250,14 @@ if __name__ == "__main__":
raise Exception
ssn_conf['aws_security_groups_ids'] =
datalab.meta_lib.get_security_group_by_name(ssn_conf['sg_name'])
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create security group for SSN",
str(err))
cleanup_aws_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
#creating roles
try:
- datalab.fab.logging_datalab.info('[CREATE ROLES]')
+ logging.info('[CREATE ROLES]')
params = "--role_name {} --role_profile_name {} --policy_name {}
--policy_file_name {} --region {} " \
"--infra_tag_name {} --infra_tag_value {} --user_tag_value
{}". \
format(ssn_conf['role_name'], ssn_conf['role_profile_name'],
ssn_conf['policy_name'],
@@ -269,14 +269,14 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create roles", str(err))
cleanup_aws_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
#creating endpoint and rout-table
try:
- datalab.fab.logging_datalab.info('[CREATE ENDPOINT AND ROUTE-TABLE]')
+ logging.info('[CREATE ENDPOINT AND ROUTE-TABLE]')
params = "--vpc_id {} --region {} --infra_tag_name {}
--infra_tag_value {}".format(
ssn_conf['aws_vpc_id'], ssn_conf['region'], ssn_conf['tag_name'],
ssn_conf['service_base_name'])
try:
@@ -285,7 +285,7 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create endpoint", str(err))
cleanup_aws_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
@@ -293,7 +293,7 @@ if __name__ == "__main__":
# creating endpoint and rout-table notebook vpc
try:
if os.environ['conf_duo_vpc_enable'] == 'true':
- datalab.fab.logging_datalab.info('[CREATE ENDPOINT AND ROUTE-TABLE
FOR NOTEBOOK VPC]')
+ logging.info('[CREATE ENDPOINT AND ROUTE-TABLE FOR NOTEBOOK VPC]')
params = "--vpc_id {} --region {} --infra_tag_name {}
--infra_tag_value {}".format(
ssn_conf['aws_vpc2_id'], ssn_conf['aws_region'],
ssn_conf['tag2_name'],
ssn_conf['service_base_name'])
@@ -303,14 +303,14 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create secondary endpoint",
str(err))
cleanup_aws_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
#creating ssn instance
try:
- datalab.fab.logging_datalab.info('[CREATE SSN INSTANCE]')
+ logging.info('[CREATE SSN INSTANCE]')
params = "--node_name {0} --ami_id {1} --instance_type {2} --key_name
{3} --security_group_ids {4} " \
"--subnet_id {5} --iam_profile {6} --infra_tag_name {7}
--infra_tag_value {8} --instance_class {9} " \
"--primary_disk_size {10}".\
@@ -324,7 +324,7 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create ssn instance", str(err))
cleanup_aws_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
@@ -332,7 +332,7 @@ if __name__ == "__main__":
#associating elastic ip
try:
if ssn_conf['network_type'] == 'public':
- datalab.fab.logging_datalab.info('[ASSOCIATING ELASTIC IP]')
+ logging.info('[ASSOCIATING ELASTIC IP]')
ssn_conf['ssn_id'] =
datalab.meta_lib.get_instance_by_name(ssn_conf['tag_name'],
ssn_conf['instance_name'])
try:
ssn_conf['elastic_ip'] = os.environ['ssn_elastic_ip']
@@ -346,7 +346,7 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create elastic ip", str(err))
cleanup_aws_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
@@ -360,7 +360,7 @@ if __name__ == "__main__":
else:
ssn_conf['instance_ip'] =
datalab.meta_lib.get_instance_ip_address(ssn_conf['tag_name'],
ssn_conf['instance_name']).get('Public')
- datalab.fab.logging_datalab.info('[CREATING ROUTE53 RECORD]')
+ logging.info('[CREATING ROUTE53 RECORD]')
try:
datalab.actions_lib.create_route_53_record(os.environ['ssn_hosted_zone_id'],
os.environ['ssn_hosted_zone_name'],
@@ -369,7 +369,7 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.logging_datalab.error('Error: {0}'.format(err))
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create route53 record", str(err))
cleanup_aws_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
