This is an automated email from the ASF dual-hosted git repository. dmysakovets pushed a commit to branch DLAB-627 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 22202c45a3d6ef1712ee6b77d322bf94bcd1bba9 Author: Dyoma33 <demian.mysakove...@gmail.com> AuthorDate: Tue Jul 16 11:24:57 2019 +0300 [DLAB-627] Created jupyter-docker templete form GCP --- .../general/files/gcp/jupyter-docker_Dockerfile | 48 ++++ .../files/gcp/jupyter-docker_description.json | 33 +++ .../scripts/gcp/jupyter-docker_configure.py | 300 +++++++++++++++++++++ 3 files changed, 381 insertions(+) diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_Dockerfile new file mode 100644 index 0000000..a44f03a --- /dev/null +++ b/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_Dockerfile @@ -0,0 +1,48 @@ +# ***************************************************************************** +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ****************************************************************************** + + +FROM docker.dlab-base:latest + +ARG OS + +COPY jupyter-docker/ /root/ +COPY general/scripts/os/* /root/scripts/ +COPY general/scripts/gcp/jupyter* /root/scripts/ +COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/dlab/notebook_lib.py +COPY general/templates/os/${OS}/ungit.service /root/templates/ +COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/ +COPY general/templates/os/pyspark_local_template.json /root/templates/ +COPY general/templates/os/py3spark_local_template.json /root/templates/ +COPY general/templates/os/pyspark_dataengine-service_template.json /root/templates/ +COPY general/templates/os/r_dataengine-service_template.json /root/templates/ +COPY general/templates/os/r_template.json /root/templates/ +COPY general/templates/os/run_template.sh /root/templates/ +COPY general/templates/os/toree_dataengine-service_* /root/templates/ +COPY general/files/os/toree-assembly-0.2.0.jar /root/files/ +COPY general/files/os/toree_kernel.tar.gz /root/files/ +COPY general/templates/os/pyspark_dataengine_template.json /root/templates/ +COPY general/templates/os/r_dataengine_template.json /root/templates/ +COPY general/templates/os/toree_dataengine_template.json /root/templates/ + +RUN chmod a+x /root/fabfile.py; \ + chmod a+x /root/scripts/* + diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_description.json b/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_description.json new file mode 100644 index 0000000..3f202f7 --- /dev/null +++ b/infrastructure-provisioning/src/general/files/gcp/jupyter-docker_description.json @@ -0,0 +1,33 @@ +{ + "exploratory_environment_shapes" : + { + "For testing" : [ + {"Size": "S", "Description": "n1-standard-2", "Type": "n1-standard-2","Ram": "7.50 GB","Cpu": "2"} + ], + "Memory optimized" : [ + {"Size": "S", "Description": "n1-highmem-4", "Type": "n1-highmem-4","Ram": "26 GB","Cpu": "4"}, + {"Size": "M", "Description": "n1-highmem-16", "Type": "n1-highmem-16","Ram": "104 GB","Cpu": "16"}, + {"Size": "L", "Description": "n1-highmem-32", "Type": "n1-highmem-32","Ram": "208 GB","Cpu": "32"} + ], + "GPU optimized": [ + {"Size": "S", "Description": "n1-standard-2", "Type": "n1-standard-2","Ram": "7.50 GB","Cpu": "2"}, + {"Size": "M", "Description": "n1-highcpu-8", "Type": "n1-highcpu-8","Ram": "7.20 GB","Cpu": "8"}, + {"Size": "L", "Description": "n1-highmem-32", "Type": "n1-highmem-32","Ram": "208 GB","Cpu": "32"} + ], + "Compute optimized": [ + {"Size": "S", "Description": "n1-highcpu-2", "Type": "n1-highcpu-2","Ram": "1.80 GB","Cpu": "2"}, + {"Size": "M", "Description": "n1-highcpu-8", "Type": "n1-highcpu-8","Ram": "7.20 GB","Cpu": "8"}, + {"Size": "L", "Description": "n1-highcpu-32", "Type": "n1-highcpu-32","Ram": "28.8 GB","Cpu": "32"} + ] + }, + "exploratory_environment_versions" : + [ + { + "template_name": "JupyterLab 0.35.6", + "description": "Base image with JupyterLab node creation routines", + "environment_type": "exploratory", + "version": "jupyter_lab-0.35.6", + "vendor": "GCP" + } + ] +} \ No newline at end of file diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter-docker_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter-docker_configure.py new file mode 100644 index 0000000..a344df9 --- /dev/null +++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter-docker_configure.py @@ -0,0 +1,300 @@ +#!/usr/bin/python + +# ***************************************************************************** +# # +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# # +# http://www.apache.org/licenses/LICENSE-2.0 +# # +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# # +# ****************************************************************************** + +# ***************************************************************************** +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ****************************************************************************** + +import logging +import json +import sys +from dlab.fab import * +from dlab.meta_lib import * +from dlab.actions_lib import * +import os + + +if __name__ == "__main__": + local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], + os.environ['request_id']) + local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename + logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s', + level=logging.DEBUG, + filename=local_log_filepath) + + notebook_config = dict() + try: + notebook_config['exploratory_name'] = (os.environ['exploratory_name']).lower().replace('_', '-') + except: + notebook_config['exploratory_name'] = '' + notebook_config['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-') + notebook_config['instance_type'] = os.environ['gcp_notebook_instance_size'] + notebook_config['key_name'] = os.environ['conf_key_name'] + notebook_config['edge_user_name'] = (os.environ['edge_user_name']).lower().replace('_', '-') + notebook_config['instance_name'] = '{0}-{1}-nb-{2}'.format(notebook_config['service_base_name'], + notebook_config['edge_user_name'], + notebook_config['exploratory_name']) + notebook_config['expected_primary_image_name'] = '{}-{}-notebook-primary-image'.format( + notebook_config['service_base_name'], os.environ['application']) + notebook_config['expected_secondary_image_name'] = '{}-{}-notebook-secondary-image'.format( + notebook_config['service_base_name'], os.environ['application']) + instance_hostname = GCPMeta().get_private_ip_address(notebook_config['instance_name']) + edge_instance_name = '{0}-{1}-edge'.format(notebook_config['service_base_name'], notebook_config['edge_user_name']) + edge_instance_hostname = GCPMeta().get_instance_public_ip_by_name(edge_instance_name) + edge_instance_private_ip = GCPMeta().get_private_ip_address(edge_instance_name) + notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) + notebook_config['dlab_ssh_user'] = os.environ['conf_os_user'] + notebook_config['zone'] = os.environ['gcp_zone'] + notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled'] + try: + if os.environ['conf_os_family'] == 'debian': + initial_user = 'ubuntu' + sudo_group = 'sudo' + if os.environ['conf_os_family'] == 'redhat': + initial_user = 'ec2-user' + sudo_group = 'wheel' + + logging.info('[CREATING DLAB SSH USER]') + print('[CREATING DLAB SSH USER]') + params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\ + (instance_hostname, notebook_config['ssh_key_path'], initial_user, + notebook_config['dlab_ssh_user'], sudo_group) + + try: + local("~/scripts/{}.py {}".format('create_ssh_user', params)) + except: + traceback.print_exc() + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed creating ssh user 'dlab'.", str(err)) + GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone']) + sys.exit(1) + + # configuring proxy on Notebook instance + try: + logging.info('[CONFIGURE PROXY ON JUPYTER-DOCKER INSTANCE]') + print('[CONFIGURE PROXY ON JUPYTER-DOCKER INSTANCE]') + additional_config = {"proxy_host": edge_instance_name, "proxy_port": "3128"} + params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}"\ + .format(instance_hostname, notebook_config['instance_name'], notebook_config['ssh_key_path'], + json.dumps(additional_config), notebook_config['dlab_ssh_user']) + try: + local("~/scripts/{}.py {}".format('common_configure_proxy', params)) + except: + traceback.print_exc() + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed to configure proxy.", str(err)) + GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone']) + sys.exit(1) + + # updating repositories & installing python packages + try: + logging.info('[INSTALLING PREREQUISITES TO JUPYTER-DOCKER NOTEBOOK INSTANCE]') + print('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]') + params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\ + format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'], + os.environ['gcp_region'], edge_instance_private_ip) + try: + local("~/scripts/{}.py {}".format('install_prerequisites', params)) + except: + traceback.print_exc() + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed installing apps: apt & pip.", str(err)) + GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone']) + sys.exit(1) + + # installing and configuring jupiter and all dependencies + try: + logging.info('[CONFIGURE JUPYTER NOTEBOOK INSTANCE]') + print('[CONFIGURE JUPYTER NOTEBOOK INSTANCE]') + params = "--hostname {} --keyfile {} " \ + "--region {} --spark_version {} " \ + "--hadoop_version {} --os_user {} " \ + "--scala_version {} --r_mirror {} " \ + "--exploratory_name {}" "--edge_ip {} ".\ + format(instance_hostname, notebook_config['ssh_key_path'], + os.environ['gcp_region'], os.environ['notebook_spark_version'], + os.environ['notebook_hadoop_version'], notebook_config['dlab_ssh_user'], + os.environ['notebook_scala_version'], os.environ['notebook_r_mirror'], + notebook_config['exploratory_name'], edge_instance_private_ip) + try: + local("~/scripts/{}.py {}".format('configure_jupyter_node', params)) + except: + traceback.print_exc() + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed to configure jupyter.", str(err)) + GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone']) + sys.exit(1) + + try: + print('[INSTALLING USERs KEY]') + logging.info('[INSTALLING USERs KEY]') + additional_config = {"user_keyname": os.environ['edge_user_name'], + "user_keydir": os.environ['conf_key_dir']} + params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format( + instance_hostname, notebook_config['ssh_key_path'], json.dumps(additional_config), notebook_config['dlab_ssh_user']) + try: + local("~/scripts/{}.py {}".format('install_user_key', params)) + except: + append_result("Failed installing users key") + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed installing users key.", str(err)) + GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone']) + sys.exit(1) + + try: + print('[SETUP USER GIT CREDENTIALS]') + logging.info('[SETUP USER GIT CREDENTIALS]') + params = '--os_user {} --notebook_ip {} --keyfile "{}"' \ + .format(notebook_config['dlab_ssh_user'], instance_hostname, notebook_config['ssh_key_path']) + try: + local("~/scripts/{}.py {}".format('common_download_git_certfile', params)) + local("~/scripts/{}.py {}".format('manage_git_creds', params)) + except: + append_result("Failed setup git credentials") + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed to setup git credentials.", str(err)) + GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone']) + sys.exit(1) + + if notebook_config['shared_image_enabled'] == 'true': + try: + print('[CREATING IMAGE]') + primary_image_id = GCPMeta().get_image_by_name(notebook_config['expected_primary_image_name']) + if primary_image_id == '': + print("Looks like it's first time we configure notebook server. Creating images.") + image_id_list = GCPActions().create_image_from_instance_disks( + notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'], + notebook_config['instance_name'], notebook_config['zone']) + if image_id_list and image_id_list[0] != '': + print("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0])) + else: + print("Looks like another image creating operation for your template have been started a moment ago.") + if image_id_list and image_id_list[1] != '': + print("Image of secondary disk was successfully created. It's ID is {}".format(image_id_list[1])) + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed creating image.", str(err)) + GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone']) + GCPActions().remove_image(notebook_config['expected_primary_image_name']) + GCPActions().remove_image(notebook_config['expected_secondary_image_name']) + sys.exit(1) + + try: + print('[SETUP EDGE REVERSE PROXY TEMPLATE]') + logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]') + additional_info = { + 'instance_hostname': instance_hostname, + 'tensor': False + } + params = "--edge_hostname {} " \ + "--keyfile {} " \ + "--os_user {} " \ + "--type {} " \ + "--exploratory_name {} " \ + "--additional_info '{}'"\ + .format(edge_instance_hostname, + notebook_config['ssh_key_path'], + notebook_config['dlab_ssh_user'], + 'jupyter', + notebook_config['exploratory_name'], + json.dumps(additional_info)) + try: + local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params)) + except: + append_result("Failed edge reverse proxy template") + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed to set edge reverse proxy template.", str(err)) + GCPActions().remove_instance(notebook_config['instance_name'], notebook_config['zone']) + sys.exit(1) + + # generating output information + ip_address = GCPMeta().get_private_ip_address(notebook_config['instance_name']) + jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name']) + ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name']) + jupyter_notebook_acces_url = "http://" + edge_instance_hostname + "/{}/".format(notebook_config['exploratory_name']) + jupyter_ungit_acces_url = "http://" + edge_instance_hostname + "/{}-ungit/".format( + notebook_config['exploratory_name']) + print('[SUMMARY]') + logging.info('[SUMMARY]') + print("Instance name: {}".format(notebook_config['instance_name'])) + print("Private IP: {}".format(ip_address)) + print("Instance type: {}".format(notebook_config['instance_type'])) + print("Key name: {}".format(notebook_config['key_name'])) + print("User key name: {}".format(os.environ['edge_user_name'])) + print("JupyterLab URL: {}".format(jupyter_ip_url)) + print("Ungit URL: {}".format(ungit_ip_url)) + print("ReverseProxyNotebook".format(jupyter_notebook_acces_url)) + print("ReverseProxyUngit".format(jupyter_ungit_acces_url)) + print('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.format(notebook_config['key_name'], + notebook_config['dlab_ssh_user'], + ip_address)) + + with open("/root/result.json", 'w') as result: + res = {"hostname": ip_address, + "ip": ip_address, + "instance_id": notebook_config['instance_name'], + "master_keyname": os.environ['conf_key_name'], + "notebook_name": notebook_config['instance_name'], + "Action": "Create new notebook server", + "exploratory_url": [ + {"description": "JupyterLab", + "url": jupyter_notebook_acces_url}, + {"description": "Ungit", + "url": jupyter_ungit_acces_url}, + {"description": "JupyterLab (via tunnel)", + "url": jupyter_ip_url}, + {"description": "Ungit (via tunnel)", + "url": ungit_ip_url} + ]} + result.write(json.dumps(res)) \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org For additional commands, e-mail: commits-h...@dlab.apache.org