This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-1107
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1107 by this push:
new 0b71619 [DLAB-1107] Added parameter for notebook image creation for
test purpose[bugfix2]
0b71619 is described below
commit 0b7161915d53375da659682cb19efd8e30e97e29
Author: Demyan Mysakovets <demian.mysakove...@gmail.com>
AuthorDate: Fri Oct 11 18:14:30 2019 +0300
[DLAB-1107] Added parameter for notebook image creation for test
purpose[bugfix2]
---
infrastructure-provisioning/scripts/deploy_dlab.py | 2 +-
.../src/general/conf/dlab.ini | 4 +--
.../src/general/lib/aws/actions_lib.py | 2 +-
.../general/scripts/aws/common_prepare_notebook.py | 23 ++++++++----
.../general/scripts/aws/deeplearning_configure.py | 42 +++++++++++++++++-----
.../src/general/scripts/aws/jupyter_configure.py | 26 ++++++++------
.../src/general/scripts/aws/rstudio_configure.py | 42 +++++++++++++++++-----
.../scripts/aws/tensor-rstudio_configure.py | 42 +++++++++++++++++-----
.../src/general/scripts/aws/tensor_configure.py | 42 +++++++++++++++++-----
.../src/general/scripts/aws/zeppelin_configure.py | 42 +++++++++++++++++-----
.../src/general/scripts/azure/jupyter_configure.py | 26 ++++++++++++--
.../src/general/scripts/gcp/jupyter_configure.py | 29 ++++++++++-----
12 files changed, 243 insertions(+), 79 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_dlab.py
b/infrastructure-provisioning/scripts/deploy_dlab.py
index 5d9a007..653c0b4 100644
--- a/infrastructure-provisioning/scripts/deploy_dlab.py
+++ b/infrastructure-provisioning/scripts/deploy_dlab.py
@@ -38,7 +38,7 @@ parser.add_argument('--conf_user_subnets_range', type=str,
default='', help='Ran
'10.10.0.0/24 - 10.10.10.0/24')
parser.add_argument('--conf_additional_tags', type=str, default='',
help='Additional tags in format '
'"Key1:Value1;Key2:Value2"')
-parser.add_argument('--conf_image_creation', type=str, default='',
help='Enable or Disable creating image at first time')
+parser.add_argument('--conf_image_enabled', type=str, default='', help='Enable
or Disable creating image at first time')#image_enabled
parser.add_argument('--conf_shared_image_enabled', type=str, default='',
help='Enable or Disable shared images')
parser.add_argument('--aws_user_predefined_s3_policies', type=str, default='',
help='Predefined policies for users '
'instances')
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini
b/infrastructure-provisioning/src/general/conf/dlab.ini
index f54b098..75fd1fe 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -45,9 +45,9 @@ pypi_mirror = pypi.doubanio.com
### Name of own GitLab SSL certificate
gitlab_certfile = dlab-gitlab.crt
###Enable or Disable creating image at first time
-#image_creation =
+# image_creation =
###Enable or Disable shared images
-#shared_image_enabled =
+# shared_image_enabled =
### CIDR of VPC
vpc_cidr = '172.31.0.0/16'
### CIDR of second VPC
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 9fb4e3c..cb928b8 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1350,7 +1350,7 @@ def create_image_from_instance(tag_name='',
instance_name='', image_name='', tag
while image.state != 'available':
local("echo Waiting for image creation; sleep 20")
image.load()
- tag = {'Key': 'Name', 'Value':
os.environ['conf_service_base_name']}
+ tag = {'Key': 'Name', 'Value': image_name},
response =
client.describe_images(ImageIds=[image.id]).get('Images')[0].get('BlockDeviceMappings')
for ebs in response:
if ebs.get('Ebs'):
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
index 64d13e8..f4cc7b1 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
@@ -75,14 +75,23 @@ if __name__ == "__main__":
os.environ['project_name'])
notebook_config['tag_name'] =
'{}-Tag'.format(notebook_config['service_base_name'])
- notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(notebook_config['service_base_name'],
-
os.environ['endpoint_name'],
+ if os.environ['conf_image_creation'] == 'true' and
os.environ['conf_shared_image_enabled'] == 'false':
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(notebook_config['service_base_name'],
+
os.environ['endpoint_name'],
+
os.environ['project_name'],
+
os.environ['application'])
+ notebook_config['notebook_image_name'] = (lambda x:
'{0}-{1}-{2}-{3}'.format(notebook_config['service_base_name'],
os.environ['project_name'],
-
os.environ['application'])
- notebook_config['notebook_image_name'] = (lambda x:
'{0}-{1}-{2}-{3}'.format(notebook_config['service_base_name'],
-
os.environ['project_name'],
-
os.environ['application'],
-
os.environ['notebook_image_name'].lower().replace('_', '-')) if (x != 'None'
and x != '')
+
os.environ['application'],
+
os.environ['notebook_image_name'].lower().replace('_', '-')) if (x !=
'None' and x != '')
+ else
notebook_config['expected_image_name'])(str(os.environ.get('notebook_image_name')))
+ else:
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-notebook-image'.format(notebook_config['service_base_name'],
+
os.environ['endpoint_name'],
+
os.environ['application'])
+ notebook_config['notebook_image_name'] = (lambda x:
'{0}-{1}-{2}'.format(notebook_config['service_base_name'],
+
os.environ['application'],
+
os.environ['notebook_image_name'].lower().replace('_', '-')) if (x !=
'None' and x != '')
else
notebook_config['expected_image_name'])(str(os.environ.get('notebook_image_name')))
print('Searching pre-configured images')
notebook_config['ami_id'] =
get_ami_id(os.environ['aws_{}_image_name'.format(os.environ['conf_os_family'])])
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
index 9028759..76ed1ff 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
@@ -57,10 +57,17 @@ if __name__ == "__main__":
notebook_config['instance_name'] =
'{}-{}-nb-{}-{}'.format(notebook_config['service_base_name'],
os.environ['project_name'],
notebook_config['exploratory_name'], args.uuid)
- notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(notebook_config['service_base_name'],
-
os.environ['endpoint_name'],
-
os.environ['project_name'],
-
os.environ['application'])
+ if os.environ['conf_shared_image_enabled'] == 'false':
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ os.environ['endpoint_name'],
+ os.environ['project_name'],
+ os.environ['application'])
+ else:
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ os.environ['endpoint_name'],
+ os.environ['application'])
notebook_config['notebook_image_name'] =
str(os.environ.get('notebook_image_name'))
notebook_config['role_profile_name'] = '{}-{}-nb-de-Profile' \
.format(notebook_config['service_base_name'].lower().replace('-',
'_'), os.environ['project_name'])
@@ -68,6 +75,7 @@ if __name__ == "__main__":
os.environ['project_name'])
notebook_config['tag_name'] =
'{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
+ notebook_config['image_creation'] = os.environ['conf_image_creation']
notebook_config['shared_image_enabled'] =
os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] =
get_instance_ip_address(notebook_config['tag_name'],
notebook_config['instance_name']).get('Private')
@@ -250,16 +258,32 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'],
notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['shared_image_enabled'] == 'true':
+ if notebook_config['image_creation'] == 'true':
try:
print('[CREATING AMI]')
- ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
- if ami_id == '':
+ ami_id =
get_ami_id_by_name(`notebook_config['expected_image_name']`)
+ if ami_id == '' and notebook_config['shared_image_enabled'] ==
'false':
print("Looks like it's first time we configure notebook
server. Creating image.")
try:
- os.environ['conf_additional_tags'] =
os.environ['conf_additional_tags'] +
';project_tag:{0};endpoint_tag:{1};'.format(os.environ['project_name'],
os.environ['endpoint_name'])
+ os.environ['conf_additional_tags'] = os.environ[
+
'conf_additional_tags'] + ';project_tag:{0};endpoint_tag:{1};'.format(
+ os.environ['project_name'],
os.environ['endpoint_name'])
+ except KeyError:
+ os.environ['conf_additional_tags'] =
'project_tag:{0};endpoint_tag:{1}'.format(
+ os.environ['project_name'],
os.environ['endpoint_name'])
+ image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
+
instance_name=notebook_config['instance_name'],
+
image_name=notebook_config['expected_image_name'])
+ if image_id != '':
+ print("Image was successfully created. It's ID is
{}".format(image_id))
+ else:
+ try:
+ os.environ['conf_additional_tags'] = os.environ[
+
'conf_additional_tags'] + ';ami:shared;endpoint_tag:{};'.format(
+ os.environ['endpoint_name'])
except KeyError:
- os.environ['conf_additional_tags'] =
'project_tag:{0};endpoint_tag:{1}'.format(os.environ['project_name'],
os.environ['endpoint_name'])
+ os.environ['conf_additional_tags'] =
'ami:shared;endpoint_tag:{}'.format(
+ os.environ['endpoint_name'])
image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
instance_name=notebook_config['instance_name'],
image_name=notebook_config['expected_image_name'])
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
index 07fc835..4685de7 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
@@ -57,10 +57,15 @@ if __name__ == "__main__":
notebook_config['instance_name'] =
'{}-{}-nb-{}-{}'.format(notebook_config['service_base_name'],
os.environ['project_name'],
notebook_config['exploratory_name'], args.uuid)
- notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(notebook_config['service_base_name'],
-
os.environ['endpoint_name'],
-
os.environ['project_name'],
-
os.environ['application'])
+ if os.environ['conf_shared_image_enabled'] == 'false':
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(notebook_config['service_base_name'],
+
os.environ['endpoint_name'],
+
os.environ['project_name'],
+
os.environ['application'])
+ else:
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-notebook-image'.format(notebook_config['service_base_name'],
+
os.environ['endpoint_name'],
+
os.environ['application'])
notebook_config['notebook_image_name'] =
str(os.environ.get('notebook_image_name'))
notebook_config['role_profile_name'] = '{}-{}-nb-de-Profile' \
.format(notebook_config['service_base_name'].lower().replace('-',
'_'), os.environ['project_name'])
@@ -71,12 +76,6 @@ if __name__ == "__main__":
notebook_config['shared_image_enabled'] =
os.environ['conf_shared_image_enabled']
notebook_config['image_creation'] = os.environ['conf_image_creation']
notebook_config['ip_address'] =
get_instance_ip_address(notebook_config['tag_name'],
notebook_config['instance_name']).get('Private')
- notebook_config['tags'] = {"Name": image_conf['full_image_name'],
- "SBN": image_conf['service_base_name'],
- "Project": image_conf['project_name'],
- "Image": image_conf['image_name'],
- "FIN": image_conf['full_image_name'],
- os.environ['conf_billing_tag_key']:
os.environ['conf_billing_tag_value']}
# generating variables regarding EDGE proxy on Notebook instance
instance_hostname = get_instance_hostname(notebook_config['tag_name'],
notebook_config['instance_name'])
@@ -276,7 +275,7 @@ if __name__ == "__main__":
try:
print('[CREATING AMI]')
ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
- if ami_id == '' && notebook_config['shared_image_enabled']:
+ if ami_id == '' and notebook_config['shared_image_enabled'] ==
'false':
print("Looks like it's first time we configure notebook
server. Creating image.")
try:
os.environ['conf_additional_tags'] =
os.environ['conf_additional_tags'] +
';project_tag:{0};endpoint_tag:{1};'.format(os.environ['project_name'],
os.environ['endpoint_name'])
@@ -288,6 +287,11 @@ if __name__ == "__main__":
if image_id != '':
print("Image was successfully created. It's ID is
{}".format(image_id))
else:
+ print("Looks like it's first time we configure notebook
server. Creating image.")
+ try:
+ os.environ['conf_additional_tags'] =
os.environ['conf_additional_tags'] +
';ami:shared;endpoint_tag:{};'.format(os.environ['endpoint_name'])
+ except KeyError:
+ os.environ['conf_additional_tags'] =
'ami:shared;endpoint_tag:{}'.format(os.environ['endpoint_name'])
image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
instance_name=notebook_config['instance_name'],
image_name=notebook_config['expected_image_name'])
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
index b1cd148..341c907 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
@@ -58,10 +58,17 @@ if __name__ == "__main__":
notebook_config['instance_name'] =
'{}-{}-nb-{}-{}'.format(notebook_config['service_base_name'],
os.environ['project_name'],
notebook_config['exploratory_name'], args.uuid)
- notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(notebook_config['service_base_name'],
-
os.environ['endpoint_name'],
-
os.environ['project_name'],
-
os.environ['application'])
+ if os.environ['conf_shared_image_enabled'] == 'false':
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ os.environ['endpoint_name'],
+ os.environ['project_name'],
+ os.environ['application'])
+ else:
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ os.environ['endpoint_name'],
+ os.environ['application'])
notebook_config['notebook_image_name'] =
str(os.environ.get('notebook_image_name'))
notebook_config['role_profile_name'] = '{}-{}-nb-de-Profile' \
.format(notebook_config['service_base_name'].lower().replace('-',
'_'), os.environ['project_name'])
@@ -70,6 +77,7 @@ if __name__ == "__main__":
notebook_config['tag_name'] =
'{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['rstudio_pass'] = id_generator()
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
+ notebook_config['image_creation'] = os.environ['conf_image_creation']
notebook_config['shared_image_enabled'] =
os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] =
get_instance_ip_address(notebook_config['tag_name'],
notebook_config['instance_name']).get('Private')
@@ -243,16 +251,32 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'],
notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['shared_image_enabled'] == 'true':
+ if notebook_config['image_creation'] == 'true':
try:
print('[CREATING AMI]')
- ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
- if ami_id == '':
+ ami_id =
get_ami_id_by_name(`notebook_config['expected_image_name']`)
+ if ami_id == '' and notebook_config['shared_image_enabled'] ==
'false':
print("Looks like it's first time we configure notebook
server. Creating image.")
try:
- os.environ['conf_additional_tags'] =
os.environ['conf_additional_tags'] +
';project_tag:{0};endpoint_tag:{1};'.format(os.environ['project_name'],
os.environ['endpoint_name'])
+ os.environ['conf_additional_tags'] = os.environ[
+
'conf_additional_tags'] + ';project_tag:{0};endpoint_tag:{1};'.format(
+ os.environ['project_name'],
os.environ['endpoint_name'])
+ except KeyError:
+ os.environ['conf_additional_tags'] =
'project_tag:{0};endpoint_tag:{1}'.format(
+ os.environ['project_name'],
os.environ['endpoint_name'])
+ image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
+
instance_name=notebook_config['instance_name'],
+
image_name=notebook_config['expected_image_name'])
+ if image_id != '':
+ print("Image was successfully created. It's ID is
{}".format(image_id))
+ else:
+ try:
+ os.environ['conf_additional_tags'] = os.environ[
+
'conf_additional_tags'] + ';ami:shared;endpoint_tag:{};'.format(
+ os.environ['endpoint_name'])
except KeyError:
- os.environ['conf_additional_tags'] =
'project_tag:{0};endpoint_tag:{1}'.format(os.environ['project_name'],
os.environ['endpoint_name'])
+ os.environ['conf_additional_tags'] =
'ami:shared;endpoint_tag:{}'.format(
+ os.environ['endpoint_name'])
image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
instance_name=notebook_config['instance_name'],
image_name=notebook_config['expected_image_name'])
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
index e9d2d44..1cd710b 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
@@ -59,10 +59,17 @@ if __name__ == "__main__":
notebook_config['instance_name'] =
'{}-{}-nb-{}-{}'.format(notebook_config['service_base_name'],
os.environ['project_name'],
notebook_config['exploratory_name'], args.uuid)
- notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(notebook_config['service_base_name'],
-
os.environ['endpoint_name'],
-
os.environ['project_name'],
-
os.environ['application'])
+ if os.environ['conf_shared_image_enabled'] == 'false':
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ os.environ['endpoint_name'],
+ os.environ['project_name'],
+ os.environ['application'])
+ else:
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ os.environ['endpoint_name'],
+ os.environ['application'])
notebook_config['notebook_image_name'] =
str(os.environ.get('notebook_image_name'))
notebook_config['role_profile_name'] = '{}-{}-nb-de-Profile' \
.format(notebook_config['service_base_name'].lower().replace('-',
'_'), os.environ['project_name'])
@@ -70,6 +77,7 @@ if __name__ == "__main__":
os.environ['project_name'])
notebook_config['tag_name'] =
'{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
+ notebook_config['image_creation'] = os.environ['conf_image_creation']
notebook_config['shared_image_enabled'] =
os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] =
get_instance_ip_address(notebook_config['tag_name'],
notebook_config['instance_name']).get('Private')
@@ -248,16 +256,32 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'],
notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['shared_image_enabled'] == 'true':
+ if notebook_config['image_creation'] == 'true':
try:
print('[CREATING AMI]')
- ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
- if ami_id == '':
+ ami_id =
get_ami_id_by_name(`notebook_config['expected_image_name']`)
+ if ami_id == '' and notebook_config['shared_image_enabled'] ==
'false':
print("Looks like it's first time we configure notebook
server. Creating image.")
try:
- os.environ['conf_additional_tags'] =
os.environ['conf_additional_tags'] +
';project_tag:{0};endpoint_tag:{1};'.format(os.environ['project_name'],
os.environ['endpoint_name'])
+ os.environ['conf_additional_tags'] = os.environ[
+
'conf_additional_tags'] + ';project_tag:{0};endpoint_tag:{1};'.format(
+ os.environ['project_name'],
os.environ['endpoint_name'])
+ except KeyError:
+ os.environ['conf_additional_tags'] =
'project_tag:{0};endpoint_tag:{1}'.format(
+ os.environ['project_name'],
os.environ['endpoint_name'])
+ image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
+
instance_name=notebook_config['instance_name'],
+
image_name=notebook_config['expected_image_name'])
+ if image_id != '':
+ print("Image was successfully created. It's ID is
{}".format(image_id))
+ else:
+ try:
+ os.environ['conf_additional_tags'] = os.environ[
+
'conf_additional_tags'] + ';ami:shared;endpoint_tag:{};'.format(
+ os.environ['endpoint_name'])
except KeyError:
- os.environ['conf_additional_tags'] =
'project_tag:{0};endpoint_tag:{1}'.format(os.environ['project_name'],
os.environ['endpoint_name'])
+ os.environ['conf_additional_tags'] =
'ami:shared;endpoint_tag:{}'.format(
+ os.environ['endpoint_name'])
image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
instance_name=notebook_config['instance_name'],
image_name=notebook_config['expected_image_name'])
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
index b63d587..2cce87f 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
@@ -59,10 +59,17 @@ if __name__ == "__main__":
notebook_config['instance_name'] =
'{}-{}-nb-{}-{}'.format(notebook_config['service_base_name'],
os.environ['project_name'],
notebook_config['exploratory_name'], args.uuid)
- notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(notebook_config['service_base_name'],
-
os.environ['endpoint_name'],
-
os.environ['project_name'],
-
os.environ['application'])
+ if os.environ['conf_shared_image_enabled'] == 'false':
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ os.environ['endpoint_name'],
+ os.environ['project_name'],
+ os.environ['application'])
+ else:
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ os.environ['endpoint_name'],
+ os.environ['application'])
notebook_config['notebook_image_name'] =
str(os.environ.get('notebook_image_name'))
notebook_config['role_profile_name'] = '{}-{}-nb-de-Profile' \
.format(notebook_config['service_base_name'].lower().replace('-',
'_'), os.environ['project_name'])
@@ -70,6 +77,7 @@ if __name__ == "__main__":
os.environ['project_name'])
notebook_config['tag_name'] =
'{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
+ notebook_config['image_creation'] = os.environ['conf_image_creation']
notebook_config['shared_image_enabled'] =
os.environ['conf_shared_image_enabled']
tag = {"Key": notebook_config['tag_name'],
"Value":
"{}-{}-subnet".format(notebook_config['service_base_name'],
os.environ['project_name'])}
@@ -243,16 +251,32 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'],
notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['shared_image_enabled'] == 'true':
+ if notebook_config['image_creation'] == 'true':
try:
print('[CREATING AMI]')
- ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
- if ami_id == '':
+ ami_id =
get_ami_id_by_name(`notebook_config['expected_image_name']`)
+ if ami_id == '' and notebook_config['shared_image_enabled'] ==
'false':
print("Looks like it's first time we configure notebook
server. Creating image.")
try:
- os.environ['conf_additional_tags'] =
os.environ['conf_additional_tags'] +
';project_tag:{0};endpoint_tag:{1};'.format(os.environ['project_name'],
os.environ['endpoint_name'])
+ os.environ['conf_additional_tags'] = os.environ[
+
'conf_additional_tags'] + ';project_tag:{0};endpoint_tag:{1};'.format(
+ os.environ['project_name'],
os.environ['endpoint_name'])
+ except KeyError:
+ os.environ['conf_additional_tags'] =
'project_tag:{0};endpoint_tag:{1}'.format(
+ os.environ['project_name'],
os.environ['endpoint_name'])
+ image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
+
instance_name=notebook_config['instance_name'],
+
image_name=notebook_config['expected_image_name'])
+ if image_id != '':
+ print("Image was successfully created. It's ID is
{}".format(image_id))
+ else:
+ try:
+ os.environ['conf_additional_tags'] = os.environ[
+
'conf_additional_tags'] + ';ami:shared;endpoint_tag:{};'.format(
+ os.environ['endpoint_name'])
except KeyError:
- os.environ['conf_additional_tags'] =
'project_tag:{0};endpoint_tag:{1}'.format(os.environ['project_name'],
os.environ['endpoint_name'])
+ os.environ['conf_additional_tags'] =
'ami:shared;endpoint_tag:{}'.format(
+ os.environ['endpoint_name'])
image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
instance_name=notebook_config['instance_name'],
image_name=notebook_config['expected_image_name'])
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
index 890025d..6a1ad93 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
@@ -59,10 +59,17 @@ if __name__ == "__main__":
notebook_config['instance_name'] =
'{}-{}-nb-{}-{}'.format(notebook_config['service_base_name'],
os.environ['project_name'],
notebook_config['exploratory_name'], args.uuid)
- notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(notebook_config['service_base_name'],
-
os.environ['endpoint_name'],
-
os.environ['project_name'],
-
os.environ['application'])
+ if os.environ['conf_shared_image_enabled'] == 'false':
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ os.environ['endpoint_name'],
+ os.environ['project_name'],
+ os.environ['application'])
+ else:
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ os.environ['endpoint_name'],
+ os.environ['application'])
notebook_config['notebook_image_name'] =
str(os.environ.get('notebook_image_name'))
notebook_config['role_profile_name'] = '{}-{}-nb-de-Profile' \
.format(notebook_config['service_base_name'].lower().replace('-',
'_'), os.environ['project_name'])
@@ -70,6 +77,7 @@ if __name__ == "__main__":
os.environ['project_name'])
notebook_config['tag_name'] =
'{}-Tag'.format(notebook_config['service_base_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
+ notebook_config['image_creation'] = os.environ['conf_image_creation']
notebook_config['shared_image_enabled'] =
os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] =
get_instance_ip_address(notebook_config['tag_name'],
notebook_config['instance_name']).get('Private')
@@ -262,16 +270,32 @@ if __name__ == "__main__":
remove_ec2(notebook_config['tag_name'],
notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['shared_image_enabled'] == 'true':
+ if notebook_config['image_creation'] == 'true':
try:
print('[CREATING AMI]')
- ami_id = get_ami_id_by_name(notebook_config['expected_image_name'])
- if ami_id == '':
+ ami_id =
get_ami_id_by_name(`notebook_config['expected_image_name']`)
+ if ami_id == '' and notebook_config['shared_image_enabled'] ==
'false':
print("Looks like it's first time we configure notebook
server. Creating image.")
try:
- os.environ['conf_additional_tags'] =
os.environ['conf_additional_tags'] +
';project_tag:{0};endpoint_tag:{1};'.format(os.environ['project_name'],
os.environ['endpoint_name'])
+ os.environ['conf_additional_tags'] = os.environ[
+
'conf_additional_tags'] + ';project_tag:{0};endpoint_tag:{1};'.format(
+ os.environ['project_name'],
os.environ['endpoint_name'])
+ except KeyError:
+ os.environ['conf_additional_tags'] =
'project_tag:{0};endpoint_tag:{1}'.format(
+ os.environ['project_name'],
os.environ['endpoint_name'])
+ image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
+
instance_name=notebook_config['instance_name'],
+
image_name=notebook_config['expected_image_name'])
+ if image_id != '':
+ print("Image was successfully created. It's ID is
{}".format(image_id))
+ else:
+ try:
+ os.environ['conf_additional_tags'] = os.environ[
+
'conf_additional_tags'] + ';ami:shared;endpoint_tag:{};'.format(
+ os.environ['endpoint_name'])
except KeyError:
- os.environ['conf_additional_tags'] =
'project_tag:{0};endpoint_tag:{1}'.format(os.environ['project_name'],
os.environ['endpoint_name'])
+ os.environ['conf_additional_tags'] =
'ami:shared;endpoint_tag:{}'.format(
+ os.environ['endpoint_name'])
image_id =
create_image_from_instance(tag_name=notebook_config['tag_name'],
instance_name=notebook_config['instance_name'],
image_name=notebook_config['expected_image_name'])
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index 845ae3a..e41346c 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@ -56,11 +56,30 @@ if __name__ == "__main__":
notebook_config['instance_name'] =
'{}-{}-nb-{}'.format(notebook_config['service_base_name'],
notebook_config['project_name'],
notebook_config['exploratory_name'])
- notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(
+ if notebook_config['shared_image_enabled'] == 'false':
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-{3}-notebook-image'.format(
notebook_config['service_base_name'],
notebook_config['endpoint_name'],
notebook_config['project_name'],
os.environ['application'])
+ notebook_config['image_tags'] = {"Name":
notebook_config['instance_name'],
+ "SBN":
notebook_config['service_base_name'],
+ "User": notebook_config['user_name'],
+ "project_tag":
notebook_config['project_tag'],
+ "endpoint_tag":
notebook_config['endpoint_tag'],
+ "Exploratory":
notebook_config['exploratory_name'],
+ os.environ['conf_billing_tag_key']:
os.environ['conf_billing_tag_value']}
+ else:
+ notebook_config['expected_image_name'] =
'{0}-{1}-{2}-notebook-image'.format(
+ notebook_config['service_base_name'],
+ notebook_config['endpoint_name'],
+ os.environ['application'])
+ notebook_config['image_tags'] = {"Name":
notebook_config['instance_name'],
+ "SBN":
notebook_config['service_base_name'],
+ "User": notebook_config['user_name'],
+ "endpoint_tag":
notebook_config['endpoint_tag'],
+ "Exploratory":
notebook_config['exploratory_name'],
+ os.environ['conf_billing_tag_key']:
os.environ['conf_billing_tag_value']}
notebook_config['notebook_image_name'] =
str(os.environ.get('notebook_image_name'))
notebook_config['security_group_name'] =
'{}-{}-nb-sg'.format(notebook_config['service_base_name'],
notebook_config['project_name'])
@@ -72,6 +91,7 @@ if __name__ == "__main__":
"endpoint_tag":
notebook_config['endpoint_tag'],
"Exploratory":
notebook_config['exploratory_name'],
os.environ['conf_billing_tag_key']:
os.environ['conf_billing_tag_value']}
+ notebook_config['image_creation'] = os.environ['conf_image_creation']
notebook_config['shared_image_enabled'] =
os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] =
AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['instance_name'])
@@ -239,7 +259,7 @@ if __name__ == "__main__":
AzureActions().remove_instance(notebook_config['resource_group_name'],
notebook_config['instance_name'])
sys.exit(1)
- if notebook_config['shared_image_enabled'] == 'true':
+ if notebook_config['image_creation'] == 'true':
try:
print('[CREATING IMAGE]')
image =
AzureMeta().get_image(notebook_config['resource_group_name'],
notebook_config['expected_image_name'])
@@ -250,7 +270,7 @@ if __name__ == "__main__":
notebook_config['instance_name'],
os.environ['azure_region'],
notebook_config['expected_image_name'],
-
json.dumps(notebook_config['tags']))
+
json.dumps(notebook_config['image_tags']))
print("Image was successfully created.")
local("~/scripts/{}.py".format('common_prepare_notebook'))
instance_running = False
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index a652f3a..5cad4f5 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -53,10 +53,23 @@ if __name__ == "__main__":
notebook_config['instance_name'] =
'{0}-{1}-nb-{2}'.format(notebook_config['service_base_name'],
notebook_config['project_name'],
notebook_config['exploratory_name'])
- notebook_config['expected_primary_image_name'] =
'{}-{}-notebook-primary-image'.format(
-
notebook_config['service_base_name'], os.environ['application'])
- notebook_config['expected_secondary_image_name'] =
'{}-{}-notebook-secondary-image'.format(
-
notebook_config['service_base_name'], os.environ['application'])
+ if notebook_config['shared_image_enabled'] == 'false':
+ notebook_config['expected_primary_image_name'] =
'{}-{}-{}-primary-image'.format(
+
notebook_config['service_base_name'], notebook_config['endpoint_tag'],
os.environ['application'])
+ notebook_config['expected_secondary_image_name'] =
'{}-{}-{}-secondary-image'.format(
+
notebook_config['service_base_name'], notebook_config['endpoint_tag'],
os.environ['application'])
+ notebook_config['image_labels'] = {"sbn":
notebook_config['service_base_name'],
+ "endpoint_tag":
notebook_config['endpoint_tag'],
+ "project_tag":
notebook_config['project_tag'],
+ "product": "dlab"}
+ else:
+ notebook_config['expected_primary_image_name'] =
'{}-{}-{}-{}-primary-image'.format(
+ notebook_config['service_base_name'],
notebook_config['endpoint_tag'], notebook_config['project_name'],
os.environ['application'])
+ notebook_config['expected_secondary_image_name'] =
'{}-{}-{}-{}-secondary-image'.format(
+ notebook_config['service_base_name'],
notebook_config['endpoint_tag'], notebook_config['project_name'],
os.environ['application'])
+ notebook_config['image_labels'] = {"sbn":
notebook_config['service_base_name'],
+ "endpoint_tag":
notebook_config['endpoint_tag'],
+ "product": "dlab"}
instance_hostname =
GCPMeta().get_private_ip_address(notebook_config['instance_name'])
edge_instance_name =
'{0}-{1}-edge'.format(notebook_config['service_base_name'],
notebook_config['project_name'])
edge_instance_hostname =
GCPMeta().get_instance_public_ip_by_name(edge_instance_name)
@@ -64,11 +77,9 @@ if __name__ == "__main__":
notebook_config['ssh_key_path'] =
'{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
+ notebook_config['image_creation'] = os.environ['conf_image_creation']
notebook_config['shared_image_enabled'] =
os.environ['conf_shared_image_enabled']
- notebook_config['image_labels'] = {"sbn":
notebook_config['service_base_name'],
- "project_tag":
notebook_config['project_tag'],
- "endpoint_tag":
notebook_config['endpoint_tag'],
- "product": "dlab"}
+
try:
if os.environ['conf_os_family'] == 'debian':
initial_user = 'ubuntu'
@@ -192,7 +203,7 @@ if __name__ == "__main__":
GCPActions().remove_instance(notebook_config['instance_name'],
notebook_config['zone'])
sys.exit(1)
- if notebook_config['shared_image_enabled'] == 'true':
+ if notebook_config['image_creation'] == 'true':
try:
print('[CREATING IMAGE]')
primary_image_id =
GCPMeta().get_image_by_name(notebook_config['expected_primary_image_name'])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org
For additional commands, e-mail: commits-h...@dlab.apache.org
