This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2409 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 1638c1d715a2c8200afcd9b2660184965639f6aa Author: leonidfrolov <[email protected]> AuthorDate: Mon Oct 11 15:42:43 2021 +0300 [DATALAB-2409]: replaced print with logging in all src/ .py scripts except in fabfiles and general/ --- .../src/base/scripts/configure_keycloak.py | 6 +- .../src/base/scripts/create_ssh_user.py | 7 +- .../src/base/scripts/install_user_key.py | 17 ++-- .../src/dataengine/scripts/configure_dataengine.py | 51 ++++++------ .../scripts/configure_deep_learning_node.py | 57 ++++++------- .../src/edge/scripts/configure_http_proxy.py | 5 +- .../edge/scripts/configure_nginx_reverse_proxy.py | 16 +--- .../src/edge/scripts/reupload_ssh_key.py | 12 +-- .../src/general/lib/aws/actions_lib.py | 2 +- .../src/general/lib/aws/meta_lib.py | 2 +- .../src/general/lib/azure/actions_lib.py | 2 +- .../src/general/lib/azure/meta_lib.py | 2 +- .../src/general/lib/gcp/actions_lib.py | 2 +- .../src/general/lib/gcp/meta_lib.py | 2 +- .../src/general/lib/os/debian/ssn_lib.py | 1 + .../src/jupyter/scripts/configure_jupyter_node.py | 51 ++++++------ .../scripts/configure_jupyterlab_node.py | 17 ++-- .../src/project/scripts/configure_http_proxy.py | 5 +- .../src/project/scripts/configure_nftables.py | 5 +- .../scripts/configure_nginx_reverse_proxy.py | 18 ++-- .../src/project/scripts/reupload_ssh_key.py | 12 +-- .../src/rstudio/scripts/configure_rstudio_node.py | 31 +++---- .../src/ssn/scripts/backup.py | 37 +++++---- .../src/ssn/scripts/configure_billing.py | 11 +-- .../src/ssn/scripts/configure_conf_file.py | 9 +- .../src/ssn/scripts/configure_docker.py | 31 +++---- .../src/ssn/scripts/configure_gitlab.py | 23 ++--- .../src/ssn/scripts/configure_mongo.py | 7 +- .../src/ssn/scripts/configure_ssn_node.py | 49 +++++------ .../src/ssn/scripts/configure_ui.py | 26 +++--- .../src/ssn/scripts/docker_build.py | 3 +- .../src/ssn/scripts/gitlab_deploy.py | 41 ++++----- .../src/ssn/scripts/resource_status.py | 5 +- .../src/ssn/scripts/restore.py | 97 +++++++++++----------- .../src/ssn/scripts/upload_response_file.py | 8 +- .../superset/scripts/configure_superset_node.py | 15 ++-- .../scripts/configure_tensor-rstudio_node.py | 37 +++++---- .../src/tensor/scripts/configure_tensor_node.py | 43 +++++----- .../zeppelin/scripts/configure_zeppelin_node.py | 51 ++++++------ 39 files changed, 404 insertions(+), 412 deletions(-) diff --git a/infrastructure-provisioning/src/base/scripts/configure_keycloak.py b/infrastructure-provisioning/src/base/scripts/configure_keycloak.py index 614b812..ecba5d6 100644 --- a/infrastructure-provisioning/src/base/scripts/configure_keycloak.py +++ b/infrastructure-provisioning/src/base/scripts/configure_keycloak.py @@ -22,7 +22,7 @@ # ****************************************************************************** import argparse -import logging +from datalab.logger import logging import requests import uuid from datalab.actions_lib import * @@ -47,7 +47,6 @@ args = parser.parse_args() ############## if __name__ == "__main__": try: - print('[CONFIGURE KEYCLOAK]') logging.info('[CONFIGURE KEYCLOAK]') keycloak_auth_server_url = '{}/realms/master/protocol/openid-connect/token'.format( args.keycloak_auth_server_url) @@ -70,7 +69,6 @@ if __name__ == "__main__": keycloak_client_id = str(uuid.uuid4()) if args.hostname == '': keycloak_redirectUris = 'https://{0}/*,http://{0}/*'.format(args.instance_public_ip).lower().split(',') - print(keycloak_redirectUris) else: keycloak_redirectUris = 'https://{0}/*,http://{0}/*,https://{1}/*,http://{1}/*'.format( args.instance_public_ip, args.hostname).lower().split(',') @@ -98,5 +96,5 @@ if __name__ == "__main__": append_result("Failed to configure keycloak.") raise Exception except Exception as err: - print('Error: {0}'.format(err)) + logging.error('Error: {0}'.format(err)) append_result("Failed to configure keycloak.", str(err)) \ No newline at end of file diff --git a/infrastructure-provisioning/src/base/scripts/create_ssh_user.py b/infrastructure-provisioning/src/base/scripts/create_ssh_user.py index 183295c..0b7c6c7 100644 --- a/infrastructure-provisioning/src/base/scripts/create_ssh_user.py +++ b/infrastructure-provisioning/src/base/scripts/create_ssh_user.py @@ -30,6 +30,7 @@ import sys import time import traceback from patchwork.files import exists +from datalab.logger import logging from patchwork import files parser = argparse.ArgumentParser() @@ -55,13 +56,13 @@ def ensure_ssh_user(initial_user, os_user, sudo_group): conn.sudo('touch /home/{}/.ssh_user_ensured'.format(initial_user)) if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.initial_user, args.keyfile) - print("Creating ssh user: {}".format(args.os_user)) + logging.info("Creating ssh user: {}".format(args.os_user)) try: ensure_ssh_user(args.initial_user, args.os_user, args.sudo_group) except Exception as err: - print('Failed to create ssh user', str(err)) + logging.error('Failed to create ssh user', str(err)) sys.exit(1) conn.close() diff --git a/infrastructure-provisioning/src/base/scripts/install_user_key.py b/infrastructure-provisioning/src/base/scripts/install_user_key.py index 2e68abd..ef29684 100644 --- a/infrastructure-provisioning/src/base/scripts/install_user_key.py +++ b/infrastructure-provisioning/src/base/scripts/install_user_key.py @@ -26,6 +26,7 @@ import json import sys import subprocess from datalab.fab import * +from datalab.logger import logging from fabric import * parser = argparse.ArgumentParser() @@ -45,38 +46,38 @@ def copy_key(config): user_key = '{}{}.pub'.format( config.get('user_keydir'), config.get('user_keyname')) - print(user_key) + logging.info(user_key) if 'user_key' not in config or config.get('user_key') == None: key = open('{0}'.format(user_key)).read() else: key = config.get('user_key') conn.sudo('echo "{0}" >> /home/{1}/.ssh/authorized_keys'.format(key, args.user)) except: - print('No user key') + logging.error('No user key') ############## # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") try: global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.user, args.keyfile) deeper_config = json.loads(args.additional_config) except: - print('Fail connection') + logging.error('Fail connection') sys.exit(2) try: - print("Ensuring safest ssh ciphers") + logging.info("Ensuring safest ssh ciphers") ensure_ciphers() except: - print('Faild to install safest ssh ciphers') + logging.error('Faild to install safest ssh ciphers') - print("Installing users key...") + logging.info("Installing users key...") try: copy_key(deeper_config) #conn.close() except: - print("Users keyfile {0} could not be found at {1}/{0}".format(args.keyfile, deeper_config['user_keydir'])) + logging.error("Users keyfile {0} could not be found at {1}/{0}".format(args.keyfile, deeper_config['user_keydir'])) sys.exit(1) diff --git a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py index 4de0ca3..1c89423 100644 --- a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py +++ b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py @@ -27,6 +27,7 @@ import sys from datalab.actions_lib import * from datalab.fab import * from datalab.notebook_lib import * +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--hostname', type=str, default='') @@ -116,12 +117,12 @@ def start_spark(os_user, master_ip, node): if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) # PREPARE DISK - print("Prepare .ensure directory") + logging.info("Prepare .ensure directory") try: if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'): conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir') @@ -129,77 +130,77 @@ if __name__ == "__main__": sys.exit(1) # INSTALL LANGUAGES - print("Install Java") + logging.info("Install Java") ensure_jre_jdk(args.os_user) if os.environ['application'] in ('jupyter', 'zeppelin'): - print("Install Scala") + logging.info("Install Scala") ensure_scala(scala_link, args.scala_version, args.os_user) if (os.environ['application'] in ('jupyter', 'zeppelin') and os.environ['notebook_r_enabled'] == 'true') \ or os.environ['application'] in ('rstudio', 'tensor-rstudio'): - print("Installing R") + logging.info("Installing R") ensure_r(args.os_user, r_libs) - print("Install Python 3 modules") + logging.info("Install Python 3 modules") ensure_python3_libraries(args.os_user) if os.environ['application'] == 'zeppelin': - print("Install python3 specific version") + logging.info("Install python3 specific version") ensure_python3_specific_version(python3_version, args.os_user) # INSTALL PYTHON IN VIRTUALENV - print("Configure Python Virtualenv") + logging.info("Configure Python Virtualenv") ensure_python_venv(python_venv_version) # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK - print("Install Spark") + logging.info("Install Spark") ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path) - print("Install storage jars") + logging.info("Install storage jars") ensure_local_jars(args.os_user, jars_dir) - print("Configure local Spark") + logging.info("Configure local Spark") configure_local_spark(jars_dir, templates_dir, '') # INSTALL TENSORFLOW AND OTHER DEEP LEARNING LIBRARIES if os.environ['application'] in ('tensor', 'tensor-rstudio', 'deeplearning'): - print("Installing TensorFlow") + logging.info("Installing TensorFlow") install_tensor(args.os_user, cuda_version, cuda_file_name, cudnn_version, cudnn_file_name, tensorflow_version, templates_dir, nvidia_version) - print("Install Theano") + logging.info("Install Theano") install_theano(args.os_user, theano_version) - print("Installing Keras") + logging.info("Installing Keras") install_keras(args.os_user, keras_version) # INSTALL DEEP LEARNING FRAMEWORKS if os.environ['application'] == 'deeplearning': - print("Installing Caffe2") + logging.info("Installing Caffe2") install_caffe2(args.os_user, caffe2_version, cmake_version) - #print("Installing Torch") + #logging.info("Installing Torch") #install_torch(args.os_user) - print("Install CNTK Python library") + logging.info("Install CNTK Python library") install_cntk(args.os_user, cntk_version) - print("Installing MXNET") + logging.info("Installing MXNET") install_mxnet(args.os_user, mxnet_version) # START SPARK CLUSTER if args.node_type == 'master': - print("Starting Spark master") + logging.info("Starting Spark master") start_spark(args.os_user, args.hostname, node='master') elif args.node_type == 'slave': - print("Starting Spark slave") + logging.info("Starting Spark slave") start_spark(args.os_user, args.master_ip, node='slave') # INSTALL OPTIONAL PACKAGES if os.environ['application'] in ('jupyter', 'zeppelin', 'tensor', 'deeplearning'): - print("Install additional Python packages") + logging.info("Install additional Python packages") ensure_additional_python_libs(args.os_user) - print("Install matplotlib") + logging.info("Install matplotlib") ensure_matplot(args.os_user) if os.environ['application'] == 'jupyter': - print("Install SBT") + logging.info("Install SBT") ensure_sbt(args.os_user) - print("Install Breeze") + logging.info("Install Breeze") add_breeze_library_local(args.os_user) if os.environ['application'] == 'zeppelin' and os.environ['notebook_r_enabled'] == 'true': - print("Install additional R packages") + logging.info("Install additional R packages") install_r_packages(args.os_user) # INSTALL LIVY diff --git a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py index 2c7a88d..f23ee12 100644 --- a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py +++ b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py @@ -30,6 +30,7 @@ from datalab.notebook_lib import * from fabric import * from patchwork.files import exists from patchwork import files +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--hostname', type=str, default='') @@ -107,22 +108,22 @@ def configure_jupyterlab_at_gcp_image(os_user, exploratory_name): if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) # PREPARE DISK - print("Prepare .ensure directory") + logging.info("Prepare .ensure directory") try: if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'): conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir') conn.sudo('touch /home/' + args.os_user + '/.ensure_dir/deep_learning') except: sys.exit(1) - print("Mount additional volume") + logging.info("Mount additional volume") if os.environ['conf_cloud_provider'] == 'gcp' and os.environ['conf_deeplearning_cloud_ami'] == 'true': - print('Additional disk premounted by google image') - print('Installing nvidia drivers') + logging.info('Additional disk premounted by google image') + logging.info('Installing nvidia drivers') try: conn.sudo('/opt/deeplearning/install-driver.sh') except: @@ -133,76 +134,76 @@ if __name__ == "__main__": if os.environ['conf_deeplearning_cloud_ami'] == 'false': # INSTALL LANGUAGES - print("Install Java") + logging.info("Install Java") ensure_jre_jdk(args.os_user) - print("Install Python 3 modules") + logging.info("Install Python 3 modules") ensure_python3_libraries(args.os_user) # INSTALL TENSORFLOW AND OTHER DEEP LEARNING LIBRARIES AND FRAMEWORKS - print("Install TensorFlow") + logging.info("Install TensorFlow") install_tensor(args.os_user, cuda_version, cuda_file_name, cudnn_version, cudnn_file_name, tensorflow_version, templates_dir, nvidia_version) - print("Install Theano") + logging.info("Install Theano") install_theano(args.os_user, theano_version) - print("Installing Keras") + logging.info("Installing Keras") install_keras(args.os_user, keras_version) - print("Installing Caffe2") + logging.info("Installing Caffe2") install_caffe2(args.os_user, caffe2_version, cmake_version) - #print("Installing Torch") + #logging.info("Installing Torch") #install_torch(args.os_user) - print("Install CNTK Python library") + logging.info("Install CNTK Python library") install_cntk(args.os_user, cntk_version) - print("Installing MXNET") + logging.info("Installing MXNET") install_mxnet(args.os_user, mxnet_version) # INSTALL JUPYTER NOTEBOOK - print("Install Jupyter") + logging.info("Install Jupyter") configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, args.jupyter_version, args.exploratory_name) # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK - print("Install local Spark") + logging.info("Install local Spark") ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path) - print("Install storage jars") + logging.info("Install storage jars") ensure_local_jars(args.os_user, jars_dir) - print("Configure local Spark") + logging.info("Configure local Spark") configure_local_spark(jars_dir, templates_dir) # INSTALL JUPYTER KERNELS - print("Install pyspark local kernel for Jupyter") + logging.info("Install pyspark local kernel for Jupyter") ensure_pyspark_local_kernel(args.os_user, pyspark_local_path_dir, templates_dir, spark_version) - print("Install py3spark local kernel for Jupyter") + logging.info("Install py3spark local kernel for Jupyter") ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version) - #print("Installing ITorch kernel for Jupyter") + #logging.info("Installing ITorch kernel for Jupyter") #install_itorch(args.os_user) # INSTALL OPTIONAL PACKAGES - print("Installing additional Python packages") + logging.info("Installing additional Python packages") ensure_additional_python_libs(args.os_user) - print("Install Matplotlib") + logging.info("Install Matplotlib") ensure_matplot(args.os_user) elif os.environ['conf_deeplearning_cloud_ami'] == 'true' and os.environ['conf_cloud_provider'] != 'gcp': # CONFIGURE JUPYTER NOTEBOOK - print("Configure Jupyter") + logging.info("Configure Jupyter") configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, args.jupyter_version, args.exploratory_name) else: configure_jupyterlab_at_gcp_image(args.os_user, args.exploratory_name) # INSTALL UNGIT - print("Install nodejs") + logging.info("Install nodejs") install_nodejs(args.os_user) - print("Install Ungit") + logging.info("Install Ungit") install_ungit(args.os_user, args.exploratory_name, args.edge_ip) if exists(conn, '/home/{0}/{1}'.format(args.os_user, gitlab_certfile)): install_gitlab_cert(args.os_user, gitlab_certfile) # INSTALL INACTIVITY CHECKER - print("Install inactivity checker") + logging.info("Install inactivity checker") install_inactivity_checker(args.os_user, args.ip_address) #POST INSTALLATION PROCESS - print("Updating pyOpenSSL library") + logging.info("Updating pyOpenSSL library") update_pyopenssl_lib(args.os_user) conn.close() \ No newline at end of file diff --git a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py index 0e9034e..a7e8e53 100644 --- a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py +++ b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py @@ -26,6 +26,7 @@ import json import sys from fabric import * from datalab.fab import * +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--hostname', type=str, default='') @@ -38,7 +39,7 @@ args = parser.parse_args() # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") try: global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.user, args.keyfile) @@ -46,6 +47,6 @@ if __name__ == "__main__": except: sys.exit(2) - print("Installing proxy for notebooks.") + logging.info("Installing proxy for notebooks.") datalab.fab.configure_http_proxy_server(deeper_config) conn.close() \ No newline at end of file diff --git a/infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py b/infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py index 6513db8..58d6a49 100644 --- a/infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py +++ b/infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py @@ -22,7 +22,7 @@ # ****************************************************************************** import argparse -import logging +from datalab.logger import logging import os import sys from datalab.edge_lib import install_nginx_ldap @@ -36,20 +36,12 @@ parser.add_argument('--user', type=str, default='') args = parser.parse_args() if __name__ == "__main__": - local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], - os.environ['edge_user_name'], - os.environ['request_id']) - local_log_filepath = "/logs/edge/" + local_log_filename - logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s', - level=logging.DEBUG, - filename=local_log_filepath) - - print("Configure connections") + logging.info("Configure connections") try: global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.user, args.keyfile) except Exception as err: - print("Failed establish connection. Excpeption: " + str(err)) + logging.error("Failed establish connection. Excpeption: " + str(err)) sys.exit(1) try: @@ -58,6 +50,6 @@ if __name__ == "__main__": os.environ['ldap_ou'], os.environ['ldap_service_password'], os.environ['ldap_service_username'], os.environ['aws_iam_user']) except Exception as err: - print("Failed install nginx reverse proxy: " + str(err)) + logging.error("Failed install nginx reverse proxy: " + str(err)) sys.exit(1) conn.close() diff --git a/infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py b/infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py index 4f8483b..0199f88 100644 --- a/infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py +++ b/infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py @@ -21,7 +21,7 @@ # # ****************************************************************************** -import logging +from datalab.logger import logging import os import sys import traceback @@ -32,18 +32,10 @@ from datalab.meta_lib import * from fabric import * if __name__ == "__main__": - local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], - os.environ['request_id']) - local_log_filepath = "/logs/edge/" + local_log_filename - logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s', - level=logging.DEBUG, - filename=local_log_filepath) - try: if os.environ['conf_cloud_provider'] == 'aws': create_aws_config_files() logging.info('[REUPLOADING USER SSH KEY]') - print('[REUPLOADING USER SSH KEY]') reupload_config = dict() reupload_config['os_user'] = os.environ['conf_os_user'] reupload_config['edge_user_name'] = os.environ['edge_user_name'] @@ -51,7 +43,7 @@ if __name__ == "__main__": reupload_config['resource_id'] = os.environ['resource_id'] reupload_config['additional_config'] = {"user_keyname": reupload_config['edge_user_name'], "user_keydir": os.environ['conf_key_dir']} - print(reupload_config) + logging.info(reupload_config) try: params = "--conf_resource {} --instance_id {} --os_user '{}'" \ " --keyfile '{}' --additional_config '{}'".format( diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py index 567c3b6..4eddcbd 100644 --- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py @@ -24,7 +24,7 @@ import backoff import boto3 import botocore import json -import logging +from datalab.logger import logging import os import sys import time diff --git a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py index 858c58d..4d8c397 100644 --- a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py +++ b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py @@ -23,7 +23,7 @@ import datalab.actions_lib import backoff import boto3 import json -import logging +from datalab.logger import logging import sys import time import traceback diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py index 756dd27..35f8072 100644 --- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py @@ -26,7 +26,7 @@ import datalab.common_lib import datalab.fab import datalab.meta_lib import json -import logging +from datalab.logger import logging import os import sys import time diff --git a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py index 510e875..821d8a9 100644 --- a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py +++ b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py @@ -31,7 +31,7 @@ from azure.datalake.store import core, lib from azure.graphrbac import GraphRbacManagementClient from azure.common.credentials import ServicePrincipalCredentials import azure.common.exceptions as AzureExceptions -import logging +from datalab.logger import logging import traceback import sys import os diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py index fa1d891..df608ea 100644 --- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py @@ -26,7 +26,7 @@ import datalab.fab import datalab.meta_lib import google.auth import json -import logging +from datalab.logger import logging import os import random import sys diff --git a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py index be5d17b..1610729 100644 --- a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py +++ b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py @@ -21,7 +21,7 @@ import backoff import google.auth -import logging +from datalab.logger import logging import os import re import sys diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py index 5439abc..81af9b1 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py @@ -29,6 +29,7 @@ import traceback from datalab.common_lib import manage_pkg from datalab.fab import * from datalab.meta_lib import * +from datalab.logger import logging from fabric import * import subprocess diff --git a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py index 3369ab2..6975be3 100644 --- a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py +++ b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py @@ -27,6 +27,7 @@ import sys from datalab.actions_lib import * from datalab.fab import * from datalab.notebook_lib import * +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--hostname', type=str, default='') @@ -71,87 +72,87 @@ gitlab_certfile = os.environ['conf_gitlab_certfile'] # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) # PREPARE DISK - print("Prepare .ensure directory") + logging.info("Prepare .ensure directory") try: if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'): conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir') except: sys.exit(1) - print("Mount additional volume") + logging.info("Mount additional volume") prepare_disk(args.os_user) # INSTALL LANGUAGES - print("Install Java") + logging.info("Install Java") ensure_jre_jdk(args.os_user) - print("Install Scala") + logging.info("Install Scala") ensure_scala(scala_link, args.scala_version, args.os_user) if os.environ['notebook_r_enabled'] == 'true': - print("Installing R") + logging.info("Installing R") ensure_r(args.os_user, r_libs) - print("Install Python 3 modules") + logging.info("Install Python 3 modules") ensure_python3_libraries(args.os_user) # INSTALL PYTHON IN VIRTUALENV - print("Configure Python Virtualenv") + logging.info("Configure Python Virtualenv") ensure_python_venv(python_venv_version) # INSTALL JUPYTER NOTEBOOK - print("Install Jupyter") + logging.info("Install Jupyter") configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, jupyter_version, args.exploratory_name) # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK - print("Install local Spark") + logging.info("Install local Spark") ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path) local_spark_scala_version = conn.run( 'export PATH=$PATH:' + local_spark_path + 'bin/; spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"').stdout.replace( '\n', '') - print("Install storage jars") + logging.info("Install storage jars") ensure_local_jars(args.os_user, jars_dir) - print("Configure local Spark") + logging.info("Configure local Spark") configure_local_spark(jars_dir, templates_dir) # INSTALL JUPYTER KERNELS - #print("Install pyspark local kernel for Jupyter") + #logging.info("Install pyspark local kernel for Jupyter") #ensure_pyspark_local_kernel(args.os_user, pyspark_local_path_dir, templates_dir, spark_version) - print("Install py3spark local kernel for Jupyter") + logging.info("Install py3spark local kernel for Jupyter") ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version, python_venv_path, python_venv_version) - print("Install Toree-Scala kernel for Jupyter") + logging.info("Install Toree-Scala kernel for Jupyter") ensure_toree_local_kernel(args.os_user, toree_link, scala_kernel_path, files_dir, local_spark_scala_version, spark_version) if os.environ['notebook_r_enabled'] == 'true': - print("Install R kernel for Jupyter") + logging.info("Install R kernel for Jupyter") ensure_r_local_kernel(spark_version, args.os_user, templates_dir, r_kernels_dir) # INSTALL UNGIT - print("Install nodejs") + logging.info("Install nodejs") install_nodejs(args.os_user) - print("Install ungit") + logging.info("Install ungit") install_ungit(args.os_user, args.exploratory_name, args.edge_ip) if exists(conn, '/home/{0}/{1}'.format(args.os_user, gitlab_certfile)): install_gitlab_cert(args.os_user, gitlab_certfile) # INSTALL INACTIVITY CHECKER - print("Install inactivity checker") + logging.info("Install inactivity checker") install_inactivity_checker(args.os_user, args.ip_address) # INSTALL OPTIONAL PACKAGES - print("Installing additional Python packages") + logging.info("Installing additional Python packages") ensure_additional_python_libs(args.os_user) - print("Install Matplotlib") + logging.info("Install Matplotlib") ensure_matplot(args.os_user) - print("Install SBT") + logging.info("Install SBT") ensure_sbt(args.os_user) - print("Install Breeze") + logging.info("Install Breeze") add_breeze_library_local(args.os_user) #POST INSTALLATION PROCESS - print("Updating pyOpenSSL library") + logging.info("Updating pyOpenSSL library") update_pyopenssl_lib(args.os_user) - print("Removing unexisting kernels") + logging.info("Removing unexisting kernels") remove_unexisting_kernel(args.os_user) conn.close() diff --git a/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py b/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py index e8b5862..3a3bd7f 100644 --- a/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py +++ b/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py @@ -27,6 +27,7 @@ import sys from datalab.actions_lib import * from datalab.fab import * from datalab.notebook_lib import * +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--hostname', type=str, default='') @@ -78,38 +79,38 @@ gitlab_certfile = os.environ['conf_gitlab_certfile'] # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) # PREPARE DISK - print("Prepare .ensure directory") + logging.info("Prepare .ensure directory") try: if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'): conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir') except: sys.exit(1) - print("Mount additional volume") + logging.info("Mount additional volume") prepare_disk(args.os_user) # INSTALL DOCKER - print ("Install Docker") + logging.info ("Install Docker") configure_docker(args.os_user) # CONFIGURE JUPYTER FILES - print("Configure jupyter files") + logging.info("Configure jupyter files") ensure_jupyterlab_files(args.os_user, jupyterlab_dir, jupyterlab_image, jupyter_conf_file, jupyterlab_conf_file, args.exploratory_name, args.edge_ip) # INSTALL UNGIT - print("Install nodejs") + logging.info("Install nodejs") install_nodejs(args.os_user) - print("Install ungit") + logging.info("Install ungit") install_ungit(args.os_user, args.exploratory_name, args.edge_ip) if exists(conn, '/home/{0}/{1}'.format(args.os_user, gitlab_certfile)): install_gitlab_cert(args.os_user, gitlab_certfile) # INSTALL INACTIVITY CHECKER - print("Install inactivity checker") + logging.info("Install inactivity checker") install_inactivity_checker(args.os_user, args.ip_address) conn.close() \ No newline at end of file diff --git a/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py b/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py index 4af93ff..ecddaf0 100644 --- a/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py +++ b/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py @@ -26,6 +26,7 @@ import json import sys from fabric import * from datalab.fab import * +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--hostname', type=str, default='') @@ -38,7 +39,7 @@ args = parser.parse_args() # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") try: global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.user, args.keyfile) @@ -46,7 +47,7 @@ if __name__ == "__main__": except: sys.exit(2) - print("Installing proxy for notebooks.") + logging.info("Installing proxy for notebooks.") datalab.fab.configure_http_proxy_server(deeper_config) conn.close() \ No newline at end of file diff --git a/infrastructure-provisioning/src/project/scripts/configure_nftables.py b/infrastructure-provisioning/src/project/scripts/configure_nftables.py index 8fe14cd..b00e228 100644 --- a/infrastructure-provisioning/src/project/scripts/configure_nftables.py +++ b/infrastructure-provisioning/src/project/scripts/configure_nftables.py @@ -26,6 +26,7 @@ import json import sys from fabric import * from datalab.fab import * +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--hostname', type=str, default='') @@ -38,7 +39,7 @@ args = parser.parse_args() # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") try: global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.user, args.keyfile) @@ -46,6 +47,6 @@ if __name__ == "__main__": except: sys.exit(2) - print("Configuring nftables on edge node.") + logging.info("Configuring nftables on edge node.") datalab.fab.configure_nftables(deeper_config) conn.close() \ No newline at end of file diff --git a/infrastructure-provisioning/src/project/scripts/configure_nginx_reverse_proxy.py b/infrastructure-provisioning/src/project/scripts/configure_nginx_reverse_proxy.py index 8baa5ce..472b252 100644 --- a/infrastructure-provisioning/src/project/scripts/configure_nginx_reverse_proxy.py +++ b/infrastructure-provisioning/src/project/scripts/configure_nginx_reverse_proxy.py @@ -22,7 +22,7 @@ # ****************************************************************************** import argparse -import logging +from datalab.logger import logging import os import sys from datalab.common_lib import ensure_step @@ -40,26 +40,18 @@ parser.add_argument('--step_cert_sans', type=str, default='') args = parser.parse_args() if __name__ == "__main__": - local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], - os.environ['project_name'], - os.environ['request_id']) - local_log_filepath = "/logs/edge/" + local_log_filename - logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s', - level=logging.DEBUG, - filename=local_log_filepath) - - print("Configure connections") + logging.info("Configure connections") try: global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.user, args.keyfile) except Exception as err: - print("Failed establish connection. Excpeption: " + str(err)) + logging.error("Failed establish connection. Excpeption: " + str(err)) sys.exit(1) if os.environ['conf_stepcerts_enabled'] == 'true': try: ensure_step(args.user) except Exception as err: - print("Failed install step: " + str(err)) + logging.error("Failed install step: " + str(err)) sys.exit(1) try: @@ -68,7 +60,7 @@ if __name__ == "__main__": args.keycloak_client_id, args.keycloak_client_secret, args.user, args.hostname, args.step_cert_sans) except Exception as err: - print("Failed install nginx reverse proxy: " + str(err)) + logging.error("Failed install nginx reverse proxy: " + str(err)) sys.exit(1) conn.close() \ No newline at end of file diff --git a/infrastructure-provisioning/src/project/scripts/reupload_ssh_key.py b/infrastructure-provisioning/src/project/scripts/reupload_ssh_key.py index 4f8483b..0199f88 100644 --- a/infrastructure-provisioning/src/project/scripts/reupload_ssh_key.py +++ b/infrastructure-provisioning/src/project/scripts/reupload_ssh_key.py @@ -21,7 +21,7 @@ # # ****************************************************************************** -import logging +from datalab.logger import logging import os import sys import traceback @@ -32,18 +32,10 @@ from datalab.meta_lib import * from fabric import * if __name__ == "__main__": - local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], - os.environ['request_id']) - local_log_filepath = "/logs/edge/" + local_log_filename - logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s', - level=logging.DEBUG, - filename=local_log_filepath) - try: if os.environ['conf_cloud_provider'] == 'aws': create_aws_config_files() logging.info('[REUPLOADING USER SSH KEY]') - print('[REUPLOADING USER SSH KEY]') reupload_config = dict() reupload_config['os_user'] = os.environ['conf_os_user'] reupload_config['edge_user_name'] = os.environ['edge_user_name'] @@ -51,7 +43,7 @@ if __name__ == "__main__": reupload_config['resource_id'] = os.environ['resource_id'] reupload_config['additional_config'] = {"user_keyname": reupload_config['edge_user_name'], "user_keydir": os.environ['conf_key_dir']} - print(reupload_config) + logging.info(reupload_config) try: params = "--conf_resource {} --instance_id {} --os_user '{}'" \ " --keyfile '{}' --additional_config '{}'".format( diff --git a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py index 397a22b..5ce8645 100644 --- a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py +++ b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py @@ -28,6 +28,7 @@ from datalab.actions_lib import * from datalab.common_lib import * from datalab.fab import * from datalab.notebook_lib import * +from datalab.logger import logging from fabric import * from patchwork.files import exists from patchwork import files @@ -67,58 +68,58 @@ gitlab_certfile = os.environ['conf_gitlab_certfile'] # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) # PREPARE DISK - print("Prepare .ensure directory") + logging.info("Prepare .ensure directory") try: if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'): conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir') except: sys.exit(1) - print("Mount additional volume") + logging.info("Mount additional volume") prepare_disk(args.os_user) # INSTALL LANGUAGES - print("Install Java") + logging.info("Install Java") ensure_jre_jdk(args.os_user) - print("Install R") + logging.info("Install R") ensure_r(args.os_user, r_libs) - print("Install Python 3 modules") + logging.info("Install Python 3 modules") ensure_python3_libraries(args.os_user) # INSTALL PYTHON IN VIRTUALENV - print("Configure Python Virtualenv") + logging.info("Configure Python Virtualenv") ensure_python_venv(python_venv_version) # INSTALL RSTUDIO - print("Install RStudio") + logging.info("Install RStudio") install_rstudio(args.os_user, local_spark_path, args.rstudio_pass, args.rstudio_version, python_venv_version) # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK - print("Install local Spark") + logging.info("Install local Spark") ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path) - print("Install storage jars") + logging.info("Install storage jars") ensure_local_jars(args.os_user, jars_dir) - print("Configure local Spark") + logging.info("Configure local Spark") configure_local_spark(jars_dir, templates_dir) # INSTALL UNGIT - print("Install nodejs") + logging.info("Install nodejs") install_nodejs(args.os_user) - print("Install Ungit") + logging.info("Install Ungit") install_ungit(args.os_user, args.exploratory_name, args.edge_ip) if exists(conn, '/home/{0}/{1}'.format(args.os_user, gitlab_certfile)): install_gitlab_cert(args.os_user, gitlab_certfile) # INSTALL INACTIVITY CHECKER - print("Install inactivity checker") + logging.info("Install inactivity checker") install_inactivity_checker(args.os_user, args.ip_address, True) #POST INSTALLATION PROCESS - print("Updating pyOpenSSL library") + logging.info("Updating pyOpenSSL library") update_pyopenssl_lib(args.os_user) conn.close() diff --git a/infrastructure-provisioning/src/ssn/scripts/backup.py b/infrastructure-provisioning/src/ssn/scripts/backup.py index dc6d91c..971551a 100644 --- a/infrastructure-provisioning/src/ssn/scripts/backup.py +++ b/infrastructure-provisioning/src/ssn/scripts/backup.py @@ -28,6 +28,7 @@ import sys import yaml from fabric import * from time import gmtime, strftime +from datalab.logger import logging import subprocess parser = argparse.ArgumentParser(description="Backup script for DataLab configs, keys, certs, jars, database & logs") @@ -72,9 +73,9 @@ def backup_prepare(): def backup_configs(): try: - print('Backup configs: {}'.format(args.configs)) + logging.info('Backup configs: {}'.format(args.configs)) if args.configs == 'skip': - print('Skipped config backup.') + logging.info('Skipped config backup.') elif args.configs == 'all': subprocess.run("find {0}{2} -name '*yml' -exec cp {3} {1}{2} \;".format(args.datalab_path, temp_folder, conf_folder, "{}"), shell=True, check=True) @@ -88,9 +89,9 @@ def backup_configs(): def backup_keys(): try: - print('Backup keys: {}'.format(args.keys)) + logging.info('Backup keys: {}'.format(args.keys)) if args.keys == 'skip': - print('Skipped keys backup.') + logging.info('Skipped keys backup.') elif args.keys == 'all': subprocess.run('cp {0}* {1}keys'.format(keys_folder, temp_folder), shell=True, check=True) else: @@ -103,9 +104,9 @@ def backup_keys(): def backup_certs(): try: - print('Backup certs: {}'.format(args.certs)) + logging.info('Backup certs: {}'.format(args.certs)) if args.certs == 'skip': - print('Skipped certs backup.') + logging.info('Skipped certs backup.') elif args.certs == 'all': for cert in all_certs: subprocess.run('sudo cp {0}{1} {2}certs'.format(certs_folder, cert, temp_folder), shell=True, check=True) @@ -121,9 +122,9 @@ def backup_certs(): def backup_jars(): try: - print('Backup jars: {}'.format(args.jars)) + logging.info('Backup jars: {}'.format(args.jars)) if args.jars == 'skip': - print('Skipped jars backup.') + logging.info('Skipped jars backup.') elif args.jars == 'all': for root, dirs, files in os.walk('{0}{1}'.format(args.datalab_path, jars_folder)): for service in dirs: @@ -138,7 +139,7 @@ def backup_jars(): def backup_database(): try: - print('Backup db: {}'.format(args.db)) + logging.info('Backup db: {}'.format(args.db)) if args.db: ssn_conf = open('{0}{1}ssn.yml'.format(args.datalab_path, conf_folder)).read() data = yaml.load('mongo{}'.format(ssn_conf.split('mongo')[-1])) @@ -153,30 +154,30 @@ def backup_database(): def backup_logs(): try: - print('Backup logs: {}'.format(args.logs)) + logging.info('Backup logs: {}'.format(args.logs)) if args.logs: - print('Backup DataLab logs') + logging.info('Backup DataLab logs') subprocess.run('cp -R {0}* {1}logs'.format(datalab_logs_folder, temp_folder), shell=True, check=True) - print('Backup docker logs') + logging.info('Backup docker logs') subprocess.run("sudo find {0} -name '*log' -exec cp {2} {1}logs/docker \;".format(docker_logs_folder, temp_folder, "{}"), shell=True, check=True) subprocess.run('sudo chown -R {0}:{0} {1}logs/docker'.format(os_user, temp_folder), shell=True, check=True) except: append_result(error='Backup logs failed.') - print('Backup logs failed.') + logging.error('Backup logs failed.') sys.exit(1) def backup_finalize(): try: - print('Compressing all files to archive...') + logging.info('Compressing all files to archive...') subprocess.run('cd {0} && tar -zcf {1} .'.format(temp_folder, dest_file), shell=True, check=True) except Exception as err: append_result(error='Compressing backup failed. {}'.format(str(err))) sys.exit(1) try: - print('Clear temp folder...') + logging.info('Clear temp folder...') if temp_folder != '/': subprocess.run('rm -rf {}'.format(temp_folder), shell=True, check=True) except Exception as err: @@ -189,12 +190,12 @@ def append_result(status='failed', error='', backup_file=''): res = {"status": status, "request_id": args.request_id} if status == 'failed': - print(error) + logging.info(error) res['error_message'] = error elif status == 'created': - print('Successfully created backup file: {}'.format(backup_file)) + logging.info('Successfully created backup file: {}'.format(backup_file)) res['backup_file'] = backup_file - print(json.dumps(res)) + logging.info(json.dumps(res)) result.write(json.dumps(res)) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py index f3357a5..7ef7380 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py @@ -24,6 +24,7 @@ import argparse import sys from fabric import * +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--cloud_provider', type=str, @@ -141,7 +142,7 @@ def yml_billing(path): f.write(config_orig) f.close() except: - print("Could not write the target file {}".format(path)) + logging.error("Could not write the target file {}".format(path)) sys.exit(1) def yml_billing_app(path): @@ -162,7 +163,7 @@ def yml_billing_app(path): f.write(config_orig) f.close() except: - print("Could not write the target file {}".format(path)) + logging.error("Could not write the target file {}".format(path)) sys.exit(1) @@ -177,7 +178,7 @@ def yml_self_service(path): f.write(config_orig) f.close() except: - print("Could not write the target file {}".format(path)) + logging.error("Could not write the target file {}".format(path)) sys.exit(1) @@ -185,7 +186,7 @@ def yml_self_service(path): # Run script # ############## if __name__ == "__main__": - print("Configure billing") + logging.info("Configure billing") # Check cloud provider # Access to the bucket without credentials? try: @@ -194,7 +195,7 @@ if __name__ == "__main__": yml_billing_app(args.datalab_dir + 'conf/billing_app.yml') yml_self_service(args.datalab_dir + 'conf/self-service.yml') except: - print('Error configure billing') + logging.error('Error configure billing') sys.exit(1) sys.exit(0) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py b/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py index dbcd025..bb7d42d 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py @@ -26,6 +26,7 @@ import argparse import json import sys from fabric import * +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--datalab_dir', type=str, default='') @@ -55,18 +56,18 @@ def modify_conf_file(): options = config.options(section) for option in options: try: - print('Trying to put variable {}_{} to conf file'.format(section, option)) + logging.info('Trying to put variable {}_{} to conf file'.format(section, option)) config.set(section, option, variables_list['{}_{}'.format(section, option)]) except: - print('Such variable doesn`t exist!') + logging.error('Such variable doesn`t exist!') config.remove_option(section, option) with open('{}sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.datalab_dir), 'w') as conf_file_final: config.write(conf_file_final) except Exception as error: - print('Error with modifying conf files:') - print(str(error)) + logging.error('Error with modifying conf files:') + logging.error(str(error)) sys.exit(1) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py index 1e6e31d..bc64c58 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py @@ -27,6 +27,7 @@ import os import sys import time from datalab.ssn_lib import * +from datalab.logger import logging from fabric import * parser = argparse.ArgumentParser() @@ -69,7 +70,7 @@ def download_toree(): conn.run('mv ./toree-0.5.0-incubating/lib/toree-assembly-0.5.0-incubating.jar {}toree-assembly-0.5.0.jar'.format(toree_path)) except Exception as err: traceback.print_exc() - print('Failed to download toree: ', str(err)) + logging.error('Failed to download toree: ', str(err)) sys.exit(1) @@ -86,7 +87,7 @@ def login_in_gcr(os_user, gcr_creds, odahu_image, datalab_path, cloud_provider): conn.sudo('apt-get -y install google-cloud-sdk') except Exception as err: traceback.print_exc() - print('Failed to install gcloud: ', str(err)) + logging.error('Failed to install gcloud: ', str(err)) sys.exit(1) try: host_string = '{}@{}'.format(args.os_user, args.hostname) @@ -99,11 +100,11 @@ def login_in_gcr(os_user, gcr_creds, odahu_image, datalab_path, cloud_provider): .format(odahu_image, datalab_path, cloud_provider)) except Exception as err: traceback.print_exc() - print('Failed to prepare odahu image: ', str(err)) + logging.error('Failed to prepare odahu image: ', str(err)) sys.exit(1) except Exception as err: traceback.print_exc() - print('Failed to prepare odahu image: ', str(err)) + logging.error('Failed to prepare odahu image: ', str(err)) sys.exit(1) def build_docker_images(image_list): @@ -182,7 +183,7 @@ def configure_guacamole(): return True except Exception as err: traceback.print_exc() - print('Failed to configure guacamole: ', str(err)) + logging.error('Failed to configure guacamole: ', str(err)) return False def status_container_removal_cron(): @@ -190,14 +191,14 @@ def status_container_removal_cron(): conn.sudo('bash -c \'echo "*/15 * * * * datalab-user docker container prune -f --filter until=50m --filter label=edge_status" >> /etc/crontab\'') except Exception as err: traceback.print_exc() - print('Failed to create admin status container removal cron: ', str(err)) + logging.error('Failed to create admin status container removal cron: ', str(err)) sys.exit(1) ############## # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") try: global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) @@ -205,34 +206,34 @@ if __name__ == "__main__": except: sys.exit(2) - print('Modifying configuration files') + logging.info('Modifying configuration files') try: modify_conf_file(args) except Exception as err: - print('Error:', str(err)) + logging.info('Error:', str(err)) sys.exit(1) - print("Downloading Apache Toree") + logging.info("Downloading Apache Toree") download_toree() - print("Installing docker daemon") + logging.info("Installing docker daemon") if not ensure_docker_daemon(args.datalab_path, args.os_user, args.region): sys.exit(1) - print("Login in Google Container Registry") + logging.info("Login in Google Container Registry") login_in_gcr(args.os_user, args.gcr_creds, args.odahu_image, args.datalab_path, args.cloud_provider) - print("Building Datalab images") + logging.info("Building Datalab images") count = 0 while not build_docker_images(deeper_config) and count < 5: count += 1 time.sleep(5) - print("Configuring guacamole") + logging.info("Configuring guacamole") if not configure_guacamole(): sys.exit(1) - print("Adding cron to remove edge status containers") + logging.info("Adding cron to remove edge status containers") status_container_removal_cron() conn.close() diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py b/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py index 59e70e1..b248932 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py @@ -28,6 +28,7 @@ import sys import subprocess from fabric import * from datalab.fab import * +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--keyfile', type=str, default='') @@ -60,7 +61,7 @@ def create_user(os_user): conn.sudo('chmod 600 /home/{0}/.ssh/authorized_keys'.format(os_user)) conn.sudo('touch /home/{}/.ssh_user_ensured'.format(initial_user)) except Exception as err: - print('Failed to install gitlab.{}'.format(str(err))) + logging.error('Failed to install gitlab.{}'.format(str(err))) sys.exit(1) conn.close() @@ -97,15 +98,15 @@ def prepare_config(): subprocess.run("sed -i 's/LDAP_ATTR_EMAIL/{}/g' gitlab.rb".format(os.environ['ldap_attr_email']), shell=True, check=True) subprocess.run("sed -i 's/GITLAB_ROOT_PASSWORD/{}/g' gitlab.rb".format(os.environ['gitlab_root_password']), shell=True, check=True) - print('Initial config is ready.') + logging.info('Initial config is ready.') except Exception as err: - print('Failed to install gitlab.{}'.format(str(err))) + logging.error('Failed to install gitlab.{}'.format(str(err))) sys.exit(1) def install_gitlab(): try: - print('Installing gitlab...') + logging.info('Installing gitlab...') if os.environ['conf_os_family'] == 'debian': conn.sudo('curl -sS https://packages.gitlab.com/install/repositories/gitlab/gitlab-ce/script.deb.sh | sudo bash') conn.sudo('apt install gitlab-ce -y') @@ -113,7 +114,7 @@ def install_gitlab(): conn.sudo('curl -sS https://packages.gitlab.com/install/repositories/gitlab/gitlab-ce/script.rpm.sh | sudo bash') conn.sudo('yum install gitlab-ce -y') else: - print('Failed to install gitlab.') + logging.info('Failed to install gitlab.') raise Exception with lcd('{}tmp/gitlab'.format(os.environ['conf_datalab_path'])): @@ -134,7 +135,7 @@ def install_gitlab(): conn.sudo('gitlab-ctl reconfigure') except Exception as err: - print('Failed to install gitlab.{}'.format(str(err))) + logging.error('Failed to install gitlab.{}'.format(str(err))) sys.exit(1) @@ -151,15 +152,15 @@ def configure_gitlab(): .format(proto, os.environ['gitlab_root_password'])).stdout.replace('\n','') data = json.loads(raw) if not json.loads(os.environ['gitlab_signup_enabled']): - print('Disabling signup...') + logging.info('Disabling signup...') conn.run('curl -k --request PUT "{0}://localhost/api/v4/application/settings?private_token={1}&sudo=root&signup_enabled=false"' .format(proto, data['private_token'])) if not json.loads(os.environ['gitlab_public_repos']): - print('Disabling public repos...') + logging.info('Disabling public repos...') conn.run('curl -k --request PUT "{0}://localhost/api/v4/application/settings?private_token={1}&sudo=root&restricted_visibility_levels=public"' .format(proto, data['private_token'])) except Exception as err: - print("Failed to connect to GitLab via API..{}".format(str(err))) + logging.error("Failed to connect to GitLab via API..{}".format(str(err))) sys.exit(1) @@ -175,9 +176,9 @@ def summary(): data['os_family'] = os.environ['conf_os_family'] data['os_user'] = os.environ['conf_os_user'] data['key_name'] = os.environ['conf_key_name'] - print('[SUMMARY]') + logging.info('[SUMMARY]') for key in data: - print('{0}: {1}'.format(key, data[key])) + logging.info('{0}: {1}'.format(key, data[key])) with open('{}tmp/result/gitlab.json'.format(os.environ['conf_datalab_path']), 'w') as result: result.write(json.dumps(data)) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py index c0d7f2f..acb9364 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py @@ -28,6 +28,7 @@ import time import yaml #from datalab.fab import * from pymongo import MongoClient +from datalab.logger import logging path = "/etc/mongod.conf" outfile = "/etc/mongo_params.yml" @@ -55,7 +56,7 @@ def add_2_yml_config(path, section, param, value): yaml.dump(config_orig, outfile_yml_w, default_flow_style=False) return True except: - print("Could not write the target file") + logging.error("Could not write the target file") return False @@ -66,7 +67,7 @@ def read_yml_conf(path, section, param): result = config[section][param] return result except: - print("File does not exist") + logging.error("File does not exist") return '' @@ -94,7 +95,7 @@ if __name__ == "__main__": command = ['service', 'mongod', 'restart'] subprocess.call(command, shell=False) except: - print("Looks like MongoDB have already been secured") + logging.error("Looks like MongoDB have already been secured") pass_upd = False # Generating output config diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py index 03ef2f7..a4d1ef3 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py @@ -29,6 +29,7 @@ import traceback from datalab.common_lib import * from datalab.fab import * from datalab.ssn_lib import * +from datalab.logger import logging from fabric import * parser = argparse.ArgumentParser() @@ -47,7 +48,7 @@ def set_hostname(subdomain, hosted_zone_name): conn.sudo('hostnamectl set-hostname {0}.{1}'.format(subdomain, hosted_zone_name)) except Exception as err: traceback.print_exc() - print('Failed to set hostname: ', str(err)) + logging.error('Failed to set hostname: ', str(err)) sys.exit(1) def set_resolve(): @@ -55,7 +56,7 @@ def set_resolve(): conn.sudo('ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf') except Exception as err: traceback.print_exc() - print('Failed to set resolve: ', str(err)) + logging.error('Failed to set resolve: ', str(err)) sys.exit(1) def cp_key(keyfile, host_string, os_user): @@ -67,7 +68,7 @@ def cp_key(keyfile, host_string, os_user): conn.sudo('chmod 600 /home/' + os_user + '/keys/*.pem') except Exception as err: traceback.print_exc() - print('Failed to copy key: ', str(err)) + logging.error('Failed to copy key: ', str(err)) sys.exit(1) @@ -78,7 +79,7 @@ def cp_backup_scripts(datalab_path): conn.run('chmod +x {0}tmp/backup.py {0}tmp/restore.py'.format(datalab_path)) except Exception as err: traceback.print_exc() - print('Failed to copy backup scripts: ', str(err)) + logging.error('Failed to copy backup scripts: ', str(err)) sys.exit(1) @@ -98,7 +99,7 @@ def cp_gitlab_scripts(datalab_path): conn.run('cd {}tmp/gitlab && sed -i "s/SERVICE_BASE_NAME/{}/g" gitlab.ini'.format(datalab_path, os.environ['conf_service_base_name'])) except Exception as err: traceback.print_exc() - print('Failed to copy gitlab scripts: ', str(err)) + logging.error('Failed to copy gitlab scripts: ', str(err)) sys.exit(1) @@ -123,7 +124,7 @@ def creating_service_directories(datalab_path, os_user): conn.sudo('chown -R ' + os_user + ':' + os_user + ' ' + datalab_path) except Exception as err: traceback.print_exc() - print('Failed to create service directories: ', str(err)) + logging.error('Failed to create service directories: ', str(err)) sys.exit(1) @@ -191,7 +192,7 @@ def configure_ssl_certs(hostname, custom_ssl_cert): conn.sudo('openssl dhparam -out /etc/ssl/certs/dhparam.pem 2048') except Exception as err: traceback.print_exc() - print('Failed to configure SSL certificates: ', str(err)) + logging.error('Failed to configure SSL certificates: ', str(err)) sys.exit(1) def docker_build_script(): @@ -201,7 +202,7 @@ def docker_build_script(): conn.sudo('mv docker_build /usr/bin/docker-build') except Exception as err: traceback.print_exc() - print('Failed to configure docker_build script: ', str(err)) + logging.error('Failed to configure docker_build script: ', str(err)) sys.exit(1) ############## @@ -210,7 +211,7 @@ def docker_build_script(): if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") try: global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) @@ -228,31 +229,31 @@ if __name__ == "__main__": else: custom_ssl_cert = False - print('Setting resolve DNS configuration') + logging.info('Setting resolve DNS configuration') set_resolve() - print("Creating service directories.") + logging.info("Creating service directories.") creating_service_directories(args.datalab_path, args.os_user) if domain_created: - print("Setting hostname") + logging.info("Setting hostname") set_hostname(os.environ['ssn_subdomain'], os.environ['ssn_hosted_zone_name']) args.hostname = "{0}.{1}".format(os.environ['ssn_subdomain'], os.environ['ssn_hosted_zone_name']) - print("Installing nginx as frontend.") + logging.info("Installing nginx as frontend.") ensure_nginx(args.datalab_path) - print("Installing Java") + logging.info("Installing Java") ensure_java(args.os_user) - print("Configuring ssl key and cert for nginx.") + logging.info("Configuring ssl key and cert for nginx.") configure_ssl_certs(args.hostname, custom_ssl_cert) - print("Configuring nginx.") + logging.info("Configuring nginx.") configure_nginx(deeper_config, args.datalab_path, args.hostname) if os.environ['conf_letsencrypt_enabled'] == 'true': - print("Configuring letsencrypt certificates.") + logging.info("Configuring letsencrypt certificates.") install_certbot(args.os_user) if 'conf_letsencrypt_email' in os.environ: run_certbot(os.environ['conf_letsencrypt_domain_name'], 'ssn', os.environ['conf_letsencrypt_email']) @@ -260,25 +261,25 @@ if __name__ == "__main__": run_certbot(os.environ['conf_letsencrypt_domain_name'], 'ssn') configure_nginx_LE(os.environ['conf_letsencrypt_domain_name'], 'ssn') - # print("Installing jenkins.") + # logging.info("Installing jenkins.") # ensure_jenkins(args.datalab_path) - # print("Configuring jenkins.") + # logging.info("Configuring jenkins.") #configure_jenkins(args.datalab_path, args.os_user, deeper_config, args.tag_resource_id) - print("Copying key") + logging.info("Copying key") cp_key(args.keyfile, host_string, args.os_user) - print("Copying backup scripts") + logging.info("Copying backup scripts") cp_backup_scripts(args.datalab_path) - print("Copying gitlab scripts & files") + logging.info("Copying gitlab scripts & files") cp_gitlab_scripts(args.datalab_path) - print("Ensuring safest ssh ciphers") + logging.info("Ensuring safest ssh ciphers") ensure_ciphers() - print("Configuring docker_build script") + logging.info("Configuring docker_build script") docker_build_script() conn.close() \ No newline at end of file diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py index d9327aa..db05276 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py @@ -23,7 +23,7 @@ import argparse import json -import logging +from datalab.logger import logging import os import sys import traceback @@ -103,7 +103,7 @@ def copy_ssn_libraries(): conn.sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab') except Exception as err: traceback.print_exc() - print('Failed to copy ssn libraries: ', str(err)) + logging.error('Failed to copy ssn libraries: ', str(err)) sys.exit(1) @@ -142,7 +142,7 @@ def configure_mongo(mongo_passwd, default_endpoint_name): args.datalab_path)) except Exception as err: traceback.print_exc() - print('Failed to configure MongoDB: ', str(err)) + logging.error('Failed to configure MongoDB: ', str(err)) sys.exit(1) @@ -179,7 +179,7 @@ def build_ui(): except: conn.run('if ! grep -w -E "(ERROR)" /tmp/maven.log > /tmp/maven_error.log; then echo "no_error" > /tmp/maven_error.log;fi') conn.run('cat /tmp/maven_error.log') - print('Failed to build Back-end: ', str(err)) + logging.error('Failed to build Back-end: ', str(err)) sys.exit(1) conn.sudo('mkdir -p {}webapp/'.format(args.datalab_path)) for service in ['self-service', 'provisioning-service', 'billing']: @@ -213,7 +213,7 @@ def build_ui(): args.datalab_path)) except Exception as err: traceback.print_exc() - print('Failed to build UI: ', str(err)) + logging.error('Failed to build UI: ', str(err)) sys.exit(1) @@ -221,7 +221,7 @@ def build_ui(): # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") try: global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) @@ -230,28 +230,28 @@ if __name__ == "__main__": except: sys.exit(2) - print("Copying DataLab libraries to SSN") + logging.info("Copying DataLab libraries to SSN") copy_ssn_libraries() - print("Installing Supervisor") + logging.info("Installing Supervisor") ensure_supervisor() - print("Installing MongoDB") + logging.info("Installing MongoDB") ensure_mongo() - print("Configuring MongoDB") + logging.info("Configuring MongoDB") configure_mongo(mongo_passwd, args.default_endpoint_name) conn.sudo('bash -c "echo DATALAB_CONF_DIR={} >> /etc/profile"'.format(datalab_conf_dir)) conn.sudo('bash -c "echo export DATALAB_CONF_DIR >> /etc/profile"') - print("Installing build dependencies for UI") + logging.info("Installing build dependencies for UI") install_build_dep() - print("Building UI") + logging.info("Building UI") build_ui() - print("Starting Self-Service(UI)") + logging.info("Starting Self-Service(UI)") start_ss(args.keyfile, host_string, datalab_conf_dir, web_path, args.os_user, mongo_passwd, keystore_passwd, args.cloud_provider, args.service_base_name, args.tag_resource_id, args.billing_tag, args.account_id, diff --git a/infrastructure-provisioning/src/ssn/scripts/docker_build.py b/infrastructure-provisioning/src/ssn/scripts/docker_build.py index f0ddafe..37c6526 100644 --- a/infrastructure-provisioning/src/ssn/scripts/docker_build.py +++ b/infrastructure-provisioning/src/ssn/scripts/docker_build.py @@ -28,6 +28,7 @@ import subprocess from fabric import * from os.path import exists from os import path +from datalab.logger import logging src_path = '/opt/datalab/sources/infrastructure-provisioning/src/' if sys.argv[1] == 'all': @@ -71,7 +72,7 @@ def image_build(src_path, node): subprocess.run('cd {3}; docker build --build-arg OS={0} --file general/files/{1}/{2}_Dockerfile -t docker.datalab-{2} .'.format( os_family, cloud_provider, node[i], src_path), shell=True, check=True) except Exception as err: - print("Failed to build {} image".format(node[i]), str(err)) + logging.error("Failed to build {} image".format(node[i]), str(err)) raise Exception except Exception as err: traceback.print_exc() diff --git a/infrastructure-provisioning/src/ssn/scripts/gitlab_deploy.py b/infrastructure-provisioning/src/ssn/scripts/gitlab_deploy.py index 813ea47..8bcdd7d 100644 --- a/infrastructure-provisioning/src/ssn/scripts/gitlab_deploy.py +++ b/infrastructure-provisioning/src/ssn/scripts/gitlab_deploy.py @@ -23,9 +23,10 @@ from ConfigParser import ConfigParser from fabric import * +from datalab.logger import logging import argparse import boto3 -from botocore.client import Config as botoConfig` +from botocore.client import Config as botoConfig import sys import os @@ -49,7 +50,7 @@ def read_ini(): if var not in os.environ: os.environ[var] = config.get(section, option) except Exception as err: - print('Failed to read conf file.{}'.format(str(err))) + logging.error('Failed to read conf file.{}'.format(str(err))) sys.exit(1) @@ -70,14 +71,14 @@ def create_instance(): InstanceType=os.environ['aws_instance_type'], SubnetId=os.environ['aws_subnet_id']) for instance in instances: - print('Waiting for instance {} become running.'.format(instance.id)) + logging.info('Waiting for instance {} become running.'.format(instance.id)) instance.wait_until_running() node_name = '{0}-{1}'.format(os.environ['conf_service_base_name'], os.environ['conf_node_name']) instance.create_tags(Tags=[{'Key': 'Name', 'Value': node_name}]) return instance.id return '' except Exception as err: - print("Failed to create instance.{}".format(str(err))) + logging.error("Failed to create instance.{}".format(str(err))) sys.exit(1) @@ -114,7 +115,7 @@ def get_ami_id(ami_name): raise Exception("Unable to find image id with name: " + ami_name) return image_id except Exception as err: - print("Failed to get AMI ID.{}".format(str(err))) + logging.error("Failed to get AMI ID.{}".format(str(err))) def create_elastic_ip(instance_id): @@ -123,9 +124,9 @@ def create_elastic_ip(instance_id): response = client.allocate_address(Domain='vpc') allocation_id = response.get('AllocationId') response = client.associate_address(InstanceId=instance_id, AllocationId=allocation_id) - print('Association ID: {}'.format(response.get('AssociationId'))) + logging.info('Association ID: {}'.format(response.get('AssociationId'))) except Exception as err: - print('Failed to allocate elastic IP.{}'.format(str(err))) + logging.error('Failed to allocate elastic IP.{}'.format(str(err))) sys.exit(1) @@ -137,7 +138,7 @@ def get_ec2_ip(instance_id): for instance in instances: return getattr(instance, 'public_dns_name') except Exception as e: - print('Failed to get instance IP.{}'.format(str(e))) + logging.error('Failed to get instance IP.{}'.format(str(e))) sys.exit(1) @@ -147,7 +148,7 @@ def put_to_bucket(bucket_name, local_file, destination_file): with open(local_file, 'rb') as data: s3.upload_fileobj(data, bucket_name, destination_file, ExtraArgs={'ServerSideEncryption': 'AES256'}) except Exception as err: - print('Unable to upload files to S3 bucket.{}'.format(str(err))) + logging.error('Unable to upload files to S3 bucket.{}'.format(str(err))) sys.exit(1) @@ -156,7 +157,7 @@ def terminate_gitlab(): ec2 = boto3.resource('ec2') client = boto3.client('ec2') node_name = '{0}-{1}'.format(os.environ['conf_service_base_name'], os.environ['conf_node_name']) - print('Terminating "{}" instance...'.format(node_name)) + logging.info('Terminating "{}" instance...'.format(node_name)) inst = ec2.instances.filter( Filters=[{'Name': 'instance-state-name', 'Values': ['running', 'stopped', 'pending', 'stopping']}, {'Name': 'tag:Name', 'Values': ['{}'.format(node_name)]}]) @@ -175,19 +176,19 @@ def terminate_gitlab(): association_id = el_ip.get('AssociationId') client.disassociate_address(AssociationId=association_id) client.release_address(AllocationId=allocation_id) - print('Releasing Elastic IP: {}'.format(elastic_ip)) + logging.info('Releasing Elastic IP: {}'.format(elastic_ip)) except: - print('There is no such Elastic IP: {}'.format(elastic_ip)) + logging.error('There is no such Elastic IP: {}'.format(elastic_ip)) except Exception as err: - print('There is no Elastic IP to disassociate from instance: {}'.format(instance.id), str(err)) + logging.error('There is no Elastic IP to disassociate from instance: {}'.format(instance.id), str(err)) client.terminate_instances(InstanceIds=[instance.id]) waiter = client.get_waiter('instance_terminated') waiter.wait(InstanceIds=[instance.id]) - print('The instance {} has been terminated successfully'.format(instance.id)) + logging.info('The instance {} has been terminated successfully'.format(instance.id)) else: - print('There are no instances with "{}" tag to terminate'.format(node_name)) + logging.info('There are no instances with "{}" tag to terminate'.format(node_name)) except Exception as err: - print('Failed to terminate gitlab instance. {}'.format(str(err))) + logging.error('Failed to terminate gitlab instance. {}'.format(str(err))) if __name__ == "__main__": @@ -196,11 +197,11 @@ if __name__ == "__main__": if args.action == 'create': instance_id = create_instance() - print('Instance {} created.'.format(instance_id)) + logging.info('Instance {} created.'.format(instance_id)) create_elastic_ip(instance_id) os.environ['instance_id'] = instance_id os.environ['instance_hostname'] = get_ec2_ip(instance_id) - print('Instance hostname: {}'.format(os.environ['instance_hostname'])) + logging.info('Instance hostname: {}'.format(os.environ['instance_hostname'])) keyfile = '{}'.format('{}{}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])) params = '--keyfile {0} --instance_ip {1}'.format(keyfile, os.environ['instance_hostname']) @@ -210,7 +211,7 @@ if __name__ == "__main__": try: subprocess.run('{0}/{1}.py {2}'.format(head, 'configure_gitlab', params), shell=True, check=True) except Exception as err: - print('Failed to configure gitlab. {}'.format(str(err))) + logging.error('Failed to configure gitlab. {}'.format(str(err))) terminate_gitlab() sys.exit(1) @@ -224,4 +225,4 @@ if __name__ == "__main__": terminate_gitlab() else: - print('Unknown action. Try again.') + logging.error('Unknown action. Try again.') diff --git a/infrastructure-provisioning/src/ssn/scripts/resource_status.py b/infrastructure-provisioning/src/ssn/scripts/resource_status.py index 7146076..2ff4a0d 100644 --- a/infrastructure-provisioning/src/ssn/scripts/resource_status.py +++ b/infrastructure-provisioning/src/ssn/scripts/resource_status.py @@ -24,6 +24,7 @@ import argparse import sys import yaml from pymongo import MongoClient +from datalab.logger import logging path = "/etc/mongod.conf" outfile = "/etc/mongo_params.yml" @@ -41,7 +42,7 @@ def read_yml_conf(path, section, param): result = config[section][param] return result except: - print("File does not exist") + logging.error("File does not exist") return '' @@ -59,5 +60,5 @@ if __name__ == "__main__": try: update_resource_status(args.resource, args.status) except: - print("Unable to update status for the resource {}".format(args.resource)) + logging.error("Unable to update status for the resource {}".format(args.resource)) sys.exit(1) \ No newline at end of file diff --git a/infrastructure-provisioning/src/ssn/scripts/restore.py b/infrastructure-provisioning/src/ssn/scripts/restore.py index 9cb9a98..b38fcc0 100644 --- a/infrastructure-provisioning/src/ssn/scripts/restore.py +++ b/infrastructure-provisioning/src/ssn/scripts/restore.py @@ -28,6 +28,7 @@ import sys import yaml import subprocess from fabric import * +from datalab.logger import logging parser = argparse.ArgumentParser(description="Restore script for DataLab configs, keys, certs, jars & database") parser.add_argument('--datalab_path', type=str, default='/opt/datalab/', help='Path to DataLab. Default: /opt/datalab/') @@ -60,7 +61,7 @@ def ask(question): else: return False except: - print("Incorrect answer. Try again...") + logging.error("Incorrect answer. Try again...") continue @@ -70,30 +71,30 @@ def restore_prepare(): head, tail = os.path.split(args.file) temp_folder = "/tmp/{}/".format(tail.split(".")[0]) if os.path.isdir(temp_folder): - print("Temporary folder with this backup already exist.") - print("Use folder path '{}' in --file key".format(temp_folder)) + logging.info("Temporary folder with this backup already exist.") + logging.info("Use folder path '{}' in --file key".format(temp_folder)) raise Exception - print("Backup acrhive will be unpacked to: {}".format(temp_folder)) + logging.info("Backup acrhive will be unpacked to: {}".format(temp_folder)) subprocess.run("mkdir {}".format(temp_folder), shell=True, check=True) subprocess.run("tar -xf {0} -C {1}".format(backup_file, temp_folder), shell=True, check=True) elif os.path.isdir(backup_file): temp_folder = backup_file else: - print("Please, specify file or folder. Try --help for more details.") + logging.info("Please, specify file or folder. Try --help for more details.") raise Exception - print("Backup acrhive: {} contains following files (exclude logs):".format(backup_file)) + logging.info("Backup acrhive: {} contains following files (exclude logs):".format(backup_file)) subprocess.run("find {} -not -name '*log'".format(temp_folder), shell=True, check=True) except Exception as err: - print("Failed to open backup.{}".format(str(err))) + logging.error("Failed to open backup.{}".format(str(err))) sys.exit(1) try: if ask("Maybe you want to create backup of existing configuration before restoring?"): with settings(hide('everything')): - print("Creating new backup...") + logging.info("Creating new backup...") subprocess.run("python3 backup.py --configs all --keys all --certs all --jar all --db", shell=True, check=True) except: - print("Failed to create new backup.") + logging.error("Failed to create new backup.") sys.exit(1) try: @@ -102,7 +103,7 @@ def restore_prepare(): else: raise Exception except: - print("Failed to stop all services. Can not continue.") + logging.error("Failed to stop all services. Can not continue.") sys.exit(1) return temp_folder @@ -111,7 +112,7 @@ def restore_prepare(): def restore_configs(): try: if not os.path.isdir("{0}{1}".format(temp_folder, conf_folder)): - print("Config files are not available in this backup.") + logging.info("Config files are not available in this backup.") raise Exception configs = list() @@ -119,12 +120,12 @@ def restore_configs(): configs = [files for root, dirs, files in os.walk("{0}{1}".format(temp_folder, conf_folder))][0] else: configs = args.configs.split(",") - print("Restore configs: {}".format(configs)) + logging.info("Restore configs: {}".format(configs)) if args.configs != "skip": for filename in configs: if not os.path.isfile("{0}{1}{2}".format(temp_folder, conf_folder, filename)): - print("Config {} are not available in this backup.".format(filename)) + logging.info("Config {} are not available in this backup.".format(filename)) else: if os.path.isfile("{0}{1}{2}".format(args.datalab_path, conf_folder, filename)): backupfile = "{0}{1}{2}".format(temp_folder, conf_folder, filename) @@ -133,20 +134,20 @@ def restore_configs(): if ask("Config {} was changed, rewrite it?".format(filename)): subprocess.run("cp -f {0} {1}".format(backupfile, destfile), shell=True, check=True) else: - print("Config {} was skipped.".format(destfile)) + logging.info("Config {} was skipped.".format(destfile)) else: - print("Config {} was not changed. Skipped.".format(filename)) + logging.info("Config {} was not changed. Skipped.".format(filename)) else: - print("Config {} does not exist. Creating.".format(filename)) + logging.info("Config {} does not exist. Creating.".format(filename)) subprocess.run("cp {0}{1}{2} {3}{1}{2}".format(temp_folder, conf_folder, filename, args.datalab_path), shell=True, check=True) except: - print("Restore configs failed.") + logging.error("Restore configs failed.") def restore_keys(): try: if not os.path.isdir("{}keys".format(temp_folder)): - print("Key files are not available in this backup.") + logging.info("Key files are not available in this backup.") raise Exception keys = list() @@ -154,33 +155,33 @@ def restore_keys(): keys = [files for root, dirs, files in os.walk("{}keys".format(temp_folder))][0] else: keys = args.keys.split(",") - print("Restore keys: {}".format(keys)) + logging.info("Restore keys: {}".format(keys)) if args.keys != "skip": for filename in keys: if not os.path.isfile("{0}keys/{1}".format(temp_folder, filename)): - print("Key {} are not available in this backup.".format(filename)) + logging.info("Key {} are not available in this backup.".format(filename)) else: if os.path.isfile("{0}{1}".format(keys_folder, filename)): - print("Key {} already exist.".format(filename)) + logging.info("Key {} already exist.".format(filename)) if not filecmp.cmp("{0}keys/{1}".format(temp_folder, filename), "{0}{1}".format(keys_folder, filename)): if ask("Key {} was changed, rewrite it?".format(filename)): subprocess.run("cp -f {0}keys/{2} {1}{2}".format(temp_folder, keys_folder, filename), shell=True, check=True) else: - print("Key {} was skipped.".format(filename)) + logging.info("Key {} was skipped.".format(filename)) else: - print("Key {} was not changed. Skipped.".format(filename)) + logging.info("Key {} was not changed. Skipped.".format(filename)) else: - print("Key {} does not exist. Creating.".format(filename)) + logging.info("Key {} does not exist. Creating.".format(filename)) subprocess.run("cp {0}keys/{2} {1}{2}".format(temp_folder, keys_folder, filename), shell=True, check=True) except: - print("Restore keys failed.") + logging.error("Restore keys failed.") def restore_certs(): try: if not os.path.isdir("{}certs".format(temp_folder)): - print("Cert files are not available in this backup.") + logging.info("Cert files are not available in this backup.") raise Exception certs = list() @@ -188,35 +189,35 @@ def restore_certs(): certs = [files for root, dirs, files in os.walk("{}certs".format(temp_folder))][0] else: certs = args.certs.split(",") - print("Restore certs: {}".format(certs)) + logging.info("Restore certs: {}".format(certs)) if args.certs != "skip": for filename in certs: if not os.path.isfile("{0}certs/{1}".format(temp_folder, filename)): - print("Cert {} are not available in this backup.".format(filename)) + logging.info("Cert {} are not available in this backup.".format(filename)) else: if os.path.isfile("{0}{1}".format(certs_folder, filename)): - print("Cert {} already exist.".format(filename)) + logging.info("Cert {} already exist.".format(filename)) if not filecmp.cmp("{0}certs/{1}".format(temp_folder, filename), "{0}{1}".format(certs_folder, filename)): if ask("Cert {} was changed, rewrite it?".format(filename)): subprocess.run("sudo cp -f {0}certs/{2} {1}{2}".format(temp_folder, certs_folder, filename), shell=True, check=True) subprocess.run("sudo chown {0}:{0} {1}{2}".format("root", certs_folder, filename), shell=True, check=True) else: - print("Cert {} was skipped.".format(filename)) + logging.info("Cert {} was skipped.".format(filename)) else: - print("Cert {} was not changed. Skipped.".format(filename)) + logging.info("Cert {} was not changed. Skipped.".format(filename)) else: - print("Cert {} does not exist. Creating.".format(filename)) + logging.info("Cert {} does not exist. Creating.".format(filename)) subprocess.run("sudo cp {0}certs/{2} {1}{2}".format(temp_folder, certs_folder, filename), shell=True, check=True) subprocess.run("sudo chown {0}:{0} {1}{2}".format("root", certs_folder, filename), shell=True, check=True) except: - print("Restore certs failed.") + logging.error("Restore certs failed.") def restore_jars(): try: if not os.path.isdir("{0}jars".format(temp_folder)): - print("Jar files are not available in this backup.") + logging.info("Jar files are not available in this backup.") raise Exception jars = list() @@ -224,12 +225,12 @@ def restore_jars(): jars = [dirs for root, dirs, files in os.walk("{}jars".format(temp_folder))][0] else: jars = args.jars.split(",") - print("Restore jars: {}".format(jars)) + logging.info("Restore jars: {}".format(jars)) if args.jars != "skip": for service in jars: if not os.path.isdir("{0}jars/{1}".format(temp_folder, service)): - print("Jar {} are not available in this backup.".format(service)) + logging.info("Jar {} are not available in this backup.".format(service)) else: for root, dirs, files in os.walk("{0}jars/{1}".format(temp_folder, service)): for filename in files: @@ -241,36 +242,36 @@ def restore_jars(): if ask("Jar {} was changed, rewrite it?".format(filename)): subprocess.run("cp -fP {0} {1}".format(backupfile, destfile), shell=True, check=True) else: - print("Jar {} was skipped.".format(destfile)) + logging.info("Jar {} was skipped.".format(destfile)) else: - print("Jar {} was not changed. Skipped.".format(filename)) + logging.info("Jar {} was not changed. Skipped.".format(filename)) else: - print("Jar {} does not exist. Creating.".format(filename)) + logging.info("Jar {} does not exist. Creating.".format(filename)) subprocess.run("cp -P {0}jars/{1}/{2} {3}{4}{1}".format(temp_folder, service, filename, args.datalab_path, jars_folder), shell=True, check=True) except: - print("Restore jars failed.") + logging.error("Restore jars failed.") def restore_database(): try: - print("Restore database: {}".format(args.db)) + logging.info("Restore database: {}".format(args.db)) if args.db: if not os.path.isfile("{0}{1}".format(temp_folder, "mongo.db")): - print("File {} are not available in this backup.".format("mongo.db")) + logging.info("File {} are not available in this backup.".format("mongo.db")) raise Exception else: if ask("Do you want to drop existing database and restore another from backup?"): ssn_conf = open(args.datalab_path + conf_folder + 'ssn.yml').read() data = yaml.load("mongo" + ssn_conf.split("mongo")[-1]) - print("Restoring database from backup") + logging.info("Restoring database from backup") subprocess.run("mongorestore --drop --host {0} --port {1} --archive={2}/mongo.db --username {3} --password '{4}' --authenticationDatabase={5}" \ .format(data['mongo']['host'], data['mongo']['port'], temp_folder, data['mongo']['username'], data['mongo']['password'], data['mongo']['database']), shell=True, check=True) else: - print("Restore database was skipped.") + logging.info("Restore database was skipped.") except: - print("Restore database failed.") + logging.error("Restore database failed.") def restore_finalize(): @@ -278,13 +279,13 @@ def restore_finalize(): if ask("Start all services after restoring?"): subprocess.run("sudo supervisorctl start all", shell=True, check=True) except: - print("Failed to start all services.") + logging.error("Failed to start all services.") try: if ask("Clean temporary folder {}?".format(temp_folder)) and temp_folder != "/": subprocess.run("rm -rf {}".format(temp_folder), shell=True, check=True) except Exception as err: - print("Clear temp folder failed. {}".format(str(err))) + logging.error("Clear temp folder failed. {}".format(str(err))) if __name__ == "__main__": @@ -308,4 +309,4 @@ if __name__ == "__main__": # Starting services & cleaning tmp folder restore_finalize() - print("Restore is finished. Good luck.") \ No newline at end of file + logging.info("Restore is finished. Good luck.") \ No newline at end of file diff --git a/infrastructure-provisioning/src/ssn/scripts/upload_response_file.py b/infrastructure-provisioning/src/ssn/scripts/upload_response_file.py index 34708c3..b61756f 100644 --- a/infrastructure-provisioning/src/ssn/scripts/upload_response_file.py +++ b/infrastructure-provisioning/src/ssn/scripts/upload_response_file.py @@ -22,7 +22,7 @@ # ****************************************************************************** import argparse -import logging +from datalab.logger import logging import sys from datalab.ssn_lib import * from datalab.fab import * @@ -36,7 +36,7 @@ args = parser.parse_args() def upload_response_file(instance_name, local_log_filepath, os_user): - print('Connect to SSN instance with hostname: {0} and name: {1}'.format(args.instance_hostname, instance_name)) + logging.info('Connect to SSN instance with hostname: {0} and name: {1}'.format(args.instance_hostname, instance_name)) pkey = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']) global conn conn = datalab.fab.init_datalab_connection(args.instance_hostname, os_user, pkey) @@ -49,12 +49,12 @@ def upload_response_file(instance_name, local_log_filepath, os_user): conn.close() return True except: - print('Failed to upload response file') + logging.error('Failed to upload response file') return False if __name__ == "__main__": - print("Uploading response file") + logging.info("Uploading response file") if not upload_response_file(args.instance_name, args.local_log_filepath, args.os_user): logging.error('Failed to upload response file') sys.exit(1) \ No newline at end of file diff --git a/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py b/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py index 5122ab4..f2a7db2 100644 --- a/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py +++ b/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py @@ -27,6 +27,7 @@ import sys from datalab.actions_lib import * from datalab.fab import * from datalab.notebook_lib import * +from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--hostname', type=str, default='') @@ -50,35 +51,35 @@ gitlab_certfile = os.environ['conf_gitlab_certfile'] # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) # PREPARE DISK - print("Prepare .ensure directory") + logging.info("Prepare .ensure directory") try: if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'): conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir') except: sys.exit(1) - #print("Mount additional volume") + #logging.info("Mount additional volume") #prepare_disk(args.os_user) # INSTALL DOCKER COMPOSE - print("Installing docker compose") + logging.info("Installing docker compose") if not ensure_docker_compose(args.os_user): sys.exit(1) # INSTALL UNGIT - print("Install nodejs") + logging.info("Install nodejs") install_nodejs(args.os_user) - print("Install ungit") + logging.info("Install ungit") install_ungit(args.os_user, args.superset_name, args.edge_instance_private_ip) if exists(conn, '/home/{0}/{1}'.format(args.os_user, gitlab_certfile)): install_gitlab_cert(args.os_user, gitlab_certfile) # INSTALL INACTIVITY CHECKER - print("Install inactivity checker") + logging.info("Install inactivity checker") install_inactivity_checker(args.os_user, args.ip_address) # PREPARE SUPERSET diff --git a/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py b/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py index e1b7cf4..4772035 100644 --- a/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py +++ b/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py @@ -28,6 +28,7 @@ from datalab.actions_lib import * from datalab.common_lib import * from datalab.fab import * from datalab.notebook_lib import * +from datalab.logger import logging from fabric import * from patchwork.files import exists from patchwork import files @@ -75,68 +76,68 @@ r_libs = ['R6', 'pbdZMQ={}'.format(os.environ['notebook_pbdzmq_version']), 'RCur # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) # PREPARE DISK - print("Prepare .ensure directory") + logging.info("Prepare .ensure directory") try: if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'): conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir') except: sys.exit(1) - print("Mount additional volume") + logging.info("Mount additional volume") prepare_disk(args.os_user) # INSTALL LANGUAGES - print("Install Java") + logging.info("Install Java") ensure_jre_jdk(args.os_user) - print("Install R") + logging.info("Install R") ensure_r(args.os_user, r_libs) - print("Install Python 3 modules") + logging.info("Install Python 3 modules") ensure_python3_libraries(args.os_user) # INSTALL PYTHON IN VIRTUALENV - print("Configure Python Virtualenv") + logging.info("Configure Python Virtualenv") ensure_python_venv(python_venv_version) # INSTALL TENSORFLOW AND OTHER DEEP LEARNING LIBRARIES - print("Install TensorFlow") + logging.info("Install TensorFlow") install_tensor(args.os_user, cuda_version, cuda_file_name, cudnn_version, cudnn_file_name, tensorflow_version, templates_dir, nvidia_version) - print("Install Theano") + logging.info("Install Theano") install_theano(args.os_user, theano_version) - print("Installing Keras") + logging.info("Installing Keras") install_keras(args.os_user, keras_version) # INSTALL RSTUDIO - print("Install RStudio") + logging.info("Install RStudio") install_rstudio(args.os_user, local_spark_path, args.rstudio_pass, args.rstudio_version, python_venv_version) # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK - print("Install local Spark") + logging.info("Install local Spark") ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path ) - print("Install storage jars") + logging.info("Install storage jars") ensure_local_jars(args.os_user, jars_dir) - print("Configure local Spark") + logging.info("Configure local Spark") configure_local_spark(jars_dir, templates_dir) # INSTALL UNGIT - print("Install nodejs") + logging.info("Install nodejs") install_nodejs(args.os_user) - print("Install Ungit") + logging.info("Install Ungit") install_ungit(args.os_user, args.exploratory_name, args.edge_ip) if exists(conn, '/home/{0}/{1}'.format(args.os_user, gitlab_certfile)): install_gitlab_cert(args.os_user, gitlab_certfile) # INSTALL INACTIVITY CHECKER - print("Install inactivity checker") + logging.info("Install inactivity checker") install_inactivity_checker(args.os_user, args.ip_address) # POST INSTALLATION PROCESS - print("Updating pyOpenSSL library") + logging.info("Updating pyOpenSSL library") update_pyopenssl_lib(args.os_user) conn.close() diff --git a/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py b/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py index c9b5e3f..6808936 100644 --- a/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py +++ b/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py @@ -28,6 +28,7 @@ from datalab.actions_lib import * from datalab.common_lib import * from datalab.fab import * from datalab.notebook_lib import * +from datalab.logger import logging from fabric import * from patchwork.files import exists from patchwork import files @@ -75,78 +76,78 @@ cudnn_file_name = os.environ['notebook_cudnn_file_name'] # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) # PREPARE DISK - print("Prepare .ensure directory") + logging.info("Prepare .ensure directory") try: if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'): conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir') except: sys.exit(1) - print("Mount additional volume") + logging.info("Mount additional volume") prepare_disk(args.os_user) # INSTALL LANGUAGES - print("Install Java") + logging.info("Install Java") ensure_jre_jdk(args.os_user) - print("Install Python 3 modules") + logging.info("Install Python 3 modules") ensure_python3_libraries(args.os_user) # INSTALL PYTHON IN VIRTUALENV - print("Configure Python Virtualenv") + logging.info("Configure Python Virtualenv") ensure_python_venv(python_venv_version) # INSTALL TENSORFLOW AND OTHER DEEP LEARNING LIBRARIES - print("Install TensorFlow") + logging.info("Install TensorFlow") install_tensor(args.os_user, cuda_version, cuda_file_name, cudnn_version, cudnn_file_name, tensorflow_version, templates_dir, nvidia_version) - print("Install Theano") + logging.info("Install Theano") install_theano(args.os_user, theano_version) - print("Installing Keras") + logging.info("Installing Keras") install_keras(args.os_user, keras_version) # INSTALL JUPYTER NOTEBOOK - print("Install Jupyter") + logging.info("Install Jupyter") configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, jupyter_version, args.exploratory_name) # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK - print("Install local Spark") + logging.info("Install local Spark") ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path ) - print("Install storage jars") + logging.info("Install storage jars") ensure_local_jars(args.os_user, jars_dir) - print("Configure local Spark") + logging.info("Configure local Spark") configure_local_spark(jars_dir, templates_dir) # INSTALL JUPYTER KERNELS - #print("Install pyspark local kernel for Jupyter") + #logging.info("Install pyspark local kernel for Jupyter") #ensure_pyspark_local_kernel(args.os_user, pyspark_local_path_dir, templates_dir, spark_version) - print("Install py3spark local kernel for Jupyter") + logging.info("Install py3spark local kernel for Jupyter") ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version, python_venv_path, python_venv_version) # INSTALL UNGIT - print("Install nodejs") + logging.info("Install nodejs") install_nodejs(args.os_user) - print("Install Ungit") + logging.info("Install Ungit") install_ungit(args.os_user, args.exploratory_name, args.edge_ip) if exists(conn, '/home/{0}/{1}'.format(args.os_user, gitlab_certfile)): install_gitlab_cert(args.os_user, gitlab_certfile) # INSTALL INACTIVITY CHECKER - print("Install inactivity checker") + logging.info("Install inactivity checker") install_inactivity_checker(args.os_user, args.ip_address) # INSTALL OPTIONAL PACKAGES - print("Installing additional Python packages") + logging.info("Installing additional Python packages") ensure_additional_python_libs(args.os_user) - print("Install Matplotlib") + logging.info("Install Matplotlib") ensure_matplot(args.os_user) #POST INSTALLATION PROCESS - print("Updating pyOpenSSL library") + logging.info("Updating pyOpenSSL library") update_pyopenssl_lib(args.os_user) conn.close() diff --git a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py index 896bc08..5709cf5 100644 --- a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py +++ b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py @@ -28,6 +28,7 @@ import sys from datalab.actions_lib import * from datalab.fab import * from datalab.notebook_lib import * +from datalab.logger import logging from fabric import * from patchwork.files import exists from patchwork import files @@ -105,7 +106,7 @@ def configure_zeppelin(os_user): conn.sudo('cp /opt/zeppelin-' + zeppelin_version + '-bin-netinst/interpreter/md/zeppelin-markdown-*.jar /opt/zeppelin/lib/interpreter/') # necessary when executing paragraph launches java process with "-cp :/opt/zeppelin/lib/interpreter/*:" conn.sudo('cp /opt/zeppelin-' + zeppelin_version + '-bin-netinst/interpreter/sh/zeppelin-shell-*.jar /opt/zeppelin/lib/interpreter/') except Exception as err: - print('Error:', str(err)) + logging.error('Error:', str(err)) sys.exit(1) try: conn.put(templates_dir + 'zeppelin-notebook.service', '/tmp/zeppelin-notebook.service') @@ -126,7 +127,7 @@ def configure_zeppelin(os_user): conn.sudo('''bash -l -c 'echo \"d /var/run/zeppelin 0755 {}\" > /usr/lib/tmpfiles.d/zeppelin.conf' '''.format(os_user)) conn.sudo('touch /home/' + os_user + '/.ensure_dir/zeppelin_ensured') except Exception as err: - print('Error:', str(err)) + logging.error('Error:', str(err)) sys.exit(1) @@ -203,64 +204,64 @@ def install_local_livy(args): # Run script # ############## if __name__ == "__main__": - print("Configure connections") + logging.info("Configure connections") global conn conn = datalab.fab.init_datalab_connection(args.hostname, args.os_user, args.keyfile) deeper_config = json.loads(args.additional_config) # PREPARE DISK - print("Prepare .ensure directory") + logging.info("Prepare .ensure directory") try: if not exists(conn,'/home/' + args.os_user + '/.ensure_dir'): conn.sudo('mkdir /home/' + args.os_user + '/.ensure_dir') except: sys.exit(1) - print("Mount additional volume") + logging.info("Mount additional volume") prepare_disk(args.os_user) # INSTALL LANGUAGES - print("Install Java") + logging.info("Install Java") ensure_jre_jdk(args.os_user) - print("Installing Scala") + logging.info("Installing Scala") ensure_scala(scala_link, args.scala_version, args.os_user) if os.environ['notebook_r_enabled'] == 'true': - print("Installing R") + logging.info("Installing R") ensure_r(args.os_user, r_libs) - print("Install Python 3 modules") + logging.info("Install Python 3 modules") ensure_python3_libraries(args.os_user) # INSTALL PYTHON IN VIRTUALENV - print("Configure Python Virtualenv") + logging.info("Configure Python Virtualenv") ensure_python_venv(python_venv_version) - #print("Install Python 3 specific version") + #logging.info("Install Python 3 specific version") #ensure_python3_specific_version(python3_version, args.os_user) # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK - print("Install local Spark") + logging.info("Install local Spark") ensure_local_spark(args.os_user, spark_link, args.spark_version, args.hadoop_version, local_spark_path) - print("Install storage jars") + logging.info("Install storage jars") ensure_local_jars(args.os_user, jars_dir) - print("Configure local Spark") + logging.info("Configure local Spark") configure_local_spark(jars_dir, templates_dir) # INSTALL ZEPPELIN - print("Install Zeppelin") + logging.info("Install Zeppelin") configure_zeppelin(args.os_user) # INSTALL ZEPPELIN KERNELS if args.multiple_clusters == 'true': - print("Installing Livy for local kernels") + logging.info("Installing Livy for local kernels") install_local_livy(args) - print("Configuring local kernels") + logging.info("Configuring local kernels") configure_local_livy_kernels(args) else: - print("Configuring local kernels") + logging.info("Configuring local kernels") configure_local_spark_kernels(args, python_venv_path) # INSTALL UNGIT - print("Install nodejs") + logging.info("Install nodejs") install_nodejs(args.os_user) - print("Install Ungit") + logging.info("Install Ungit") install_ungit(args.os_user, args.exploratory_name, args.edge_ip) if exists(conn, '/home/{0}/{1}'.format(args.os_user, gitlab_certfile)): install_gitlab_cert(args.os_user, gitlab_certfile) @@ -268,20 +269,20 @@ if __name__ == "__main__": conn.sudo('cp /home/{}/.git/templates/hooks/pre-commit /opt/zeppelin/notebook/.git/hooks/'.format(args.os_user)) # INSTALL INACTIVITY CHECKER - print("Install inactivity checker") + logging.info("Install inactivity checker") install_inactivity_checker(args.os_user, args.ip_address) # INSTALL OPTIONAL PACKAGES if os.environ['notebook_r_enabled'] == 'true': - print("Install additional R packages") + logging.info("Install additional R packages") install_r_packages(args.os_user) - print("Install additional Python packages") + logging.info("Install additional Python packages") ensure_additional_python_libs(args.os_user) - print("Install Matplotlib.") + logging.info("Install Matplotlib.") ensure_matplot(args.os_user) #POST INSTALLATION PROCESS - print("Updating pyOpenSSL library") + logging.info("Updating pyOpenSSL library") update_pyopenssl_lib(args.os_user) conn.close() --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
