This is an automated email from the ASF dual-hosted git repository. dmysakovets pushed a commit to branch DLAB-433 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-433 by this push: new b08f43a [DLAB-623] Removed Spark installation and added it to Dockerfile_jupyter [bugfixing and refactoring] b08f43a is described below commit b08f43a5a20b891ce62c3ead27563160a90af3f3 Author: Dyoma33 <demian.mysakove...@gmail.com> AuthorDate: Tue May 14 10:44:14 2019 +0300 [DLAB-623] Removed Spark installation and added it to Dockerfile_jupyter [bugfixing and refactoring] --- infrastructure-provisioning/src/general/lib/os/fab.py | 7 +------ .../src/general/scripts/os/jupyter_container_start.py | 3 +-- infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter | 2 +- .../src/jupyter-docker/scripts/configure_jupyter-docker_node.py | 4 ++-- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py index fb21896..3d2894a 100644 --- a/infrastructure-provisioning/src/general/lib/os/fab.py +++ b/infrastructure-provisioning/src/general/lib/os/fab.py @@ -213,17 +213,12 @@ def configure_docker(os_user, http_file, https_file): sudo('usermod -a -G docker ' + os_user) sudo('update-rc.d docker defaults') sudo('update-rc.d docker enable') - sudo('wget https://raw.githubusercontent.com/CWSpear/local-persist/master/scripts/install.sh && chmod +x install.sh') - sudo('sed -i "66s/curl/sudo curl/g" install.sh') - sudo('sed -i "s/sudo curl/sudo -E curl/g" install.sh') - run('sudo -E ./install.sh') - sudo('rm install.sh') sudo('touch /home/{}/.ensure_dir/docker_ensured'.format(os_user)) except Exception as err: print('Failed to configure Docker:', str(err)) sys.exit(1) -def ensure_jupyter_docker_files(os_user, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, exploratory_name): +def ensure_jupyter_docker_files(os_user, templates_dir, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, exploratory_name): if not exists(jupyter_dir): try: sudo('mkdir {}'.format(jupyter_dir)) diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py index 29f146a..1412e1c 100644 --- a/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py +++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_container_start.py @@ -38,9 +38,8 @@ jupyter_dir = '/home/' + args.os_user + '/.jupyter/' def start_jupyter_container(jupyter_dir): try: with cd('{}'.format(jupyter_dir)): - run('docker volume create -d local-persist -o mountpoint=/opt --name=jup_volume') run('docker build --file /home/{}/.jupyter/Dockerfile_jupyter -t jupyter-notebook .'.format(args.os_user)) - run('docker run -d -p 8888:8888 -v jup_volume:/opt/ jupyter-notebook:latest') + run('docker run -d -p 8888:8888 jupyter-notebook:latest') except: sys.exit(1) if __name__ == "__main__": diff --git a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter index 77d3048..9582721 100644 --- a/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter +++ b/infrastructure-provisioning/src/jupyter-docker/Dockerfile_jupyter @@ -29,7 +29,7 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - tk-dev wget gnupg dirmngr build-essential openssl \ + tk-dev wget curl gnupg dirmngr build-essential openssl \ libreadline-gplv2-dev libncursesw5-dev libssl-dev \ libsqlite3-dev libgdbm-dev libc6-dev libbz2-dev \ python python-dev python-pip python-virtualenv \ diff --git a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py index aee28e8..e0ebe7f 100644 --- a/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py +++ b/infrastructure-provisioning/src/jupyter-docker/scripts/configure_jupyter-docker_node.py @@ -52,11 +52,11 @@ else: spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/spark-" + spark_version + \ "-bin-hadoop" + hadoop_version + ".tgz" -spark_script = jupyter_dir + 'spark.sh' docker_version = '18.09.4' http_file = '/etc/systemd/system/docker.service.d/http-proxy.conf' https_file = '/etc/systemd/system/docker.service.d/https-proxy.conf' jupyter_dir = '/home/' + args.os_user + '/.jupyter/' +spark_script = jupyter_dir + 'spark.sh' pyspark_local_path_dir = '/home/' + args.os_user + '/.jupyter/kernels/pyspark_local/' py3spark_local_path_dir = '/home/' + args.os_user + '/.jupyter/kernels/py3spark_local/' jupyter_conf_file = jupyter_dir + 'jupyter_notebook_config.py' @@ -104,7 +104,7 @@ if __name__ == "__main__": # CONFIGURE JUPYTER FILES print("Configure jupyter files") - ensure_jupyter_docker_files(args.os_user, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, args.exploratory_name) + ensure_jupyter_docker_files(args.os_user, templates_dir, jupyter_dir, spark_script, jupyter_conf_file, jupyter_version, docker_jupyter_conf, args.exploratory_name) # INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK # print("Install local Spark") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org For additional commands, e-mail: commits-h...@dlab.apache.org