This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 503d898 SUBMARINE-697. Provide an environment whose image includes
CUDA
503d898 is described below
commit 503d8981374a746cc9be4fa30ac662f46ce7673e
Author: Charles <[email protected]>
AuthorDate: Thu Apr 15 11:18:36 2021 +0800
SUBMARINE-697. Provide an environment whose image includes CUDA
### What is this PR for?
Add an notebook environment which include CUDA v10.2
### What type of PR is it?
[Improvement]
### Todos
* [ ] - Task
### What is the Jira issue?
[SUBMARINE-697](https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-697?filter=myopenissues)
### How should this be tested?
[Travic
CI](https://travis-ci.org/github/charlie0220/submarine/builds/764863993)
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Charles <[email protected]>
Signed-off-by: Liu Xun <[email protected]>
Closes #546 from charlie0220/SUBMARINE-697 and squashes the following
commits:
454e8b8 [Charles] Delete replicated word in database.
b33f883 [Charles] Change image name from "cuda" to "gpu"
29a00f6 [Charles] SUBMARINE-697. Provide an environment whose image
includes CUDA
---
dev-support/database/submarine-data.sql | 3 +-
dev-support/docker-images/jupyter-gpu/Dockerfile | 120 +++++++++++++++++++++
dev-support/docker-images/jupyter-gpu/build.sh | 32 ++++++
.../docker-images/jupyter-gpu/start-notebook.sh | 46 ++++++++
4 files changed, 200 insertions(+), 1 deletion(-)
diff --git a/dev-support/database/submarine-data.sql
b/dev-support/database/submarine-data.sql
index 3130896..19df463 100644
--- a/dev-support/database/submarine-data.sql
+++ b/dev-support/database/submarine-data.sql
@@ -85,7 +85,8 @@ INSERT INTO `params` (`id`, `key`, `value`, `worker_index`)
VALUES
-- Records of environment
-- ----------------------------
INSERT INTO `environment` VALUES
-('environment_1600862964725_0001', 'notebook-env',
'{"name":"notebook-env","dockerImage":"apache/submarine:jupyter-notebook-0.6.0-SNAPSHOT","kernelSpec":{"name":"submarine_jupyter_py3","channels":["defaults"],"condaDependencies":[],"pipDependencies":[]}}',
'admin', '2020-09-21 14:00:05', 'admin', '2020-09-21 14:00:14');
+('environment_1600862964725_0001', 'notebook-env',
'{"name":"notebook-env","dockerImage":"apache/submarine:jupyter-notebook-0.6.0-SNAPSHOT","kernelSpec":{"name":"submarine_jupyter_py3","channels":["defaults"],"condaDependencies":[],"pipDependencies":[]}}',
'admin', '2020-09-21 14:00:05', 'admin', '2020-09-21 14:00:14'),
+('environment_1600862964725_0002', 'notebook-gpu-env',
'{"name":"notebook-gpu-env","dockerImage":"apache/submarine:jupyter-notebook-gpu-0.6.0-SNAPSHOT","kernelSpec":{"name":"submarine_jupyter_py3","channels":["defaults"],"condaDependencies":[],"pipDependencies":[]}}',
'admin', '2021-03-28 17:00:00', 'admin', '2021-03-28 17:00:00');
-- ----------------------------
-- Records of experiment_templates
diff --git a/dev-support/docker-images/jupyter-gpu/Dockerfile
b/dev-support/docker-images/jupyter-gpu/Dockerfile
new file mode 100644
index 0000000..0071d65
--- /dev/null
+++ b/dev-support/docker-images/jupyter-gpu/Dockerfile
@@ -0,0 +1,120 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM nvidia/cuda:10.2-base-ubuntu18.04
+
+ARG NB_USER="jovyan"
+ARG NB_UID="1000"
+ARG NB_PREFIX="/"
+ARG NB_PORT=8888
+ARG MLFLOW_TRACKING_URI="http://10.96.0.3:8080";
+
+USER root
+
+SHELL ["/bin/bash", "-c"]
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq
--no-install-recommends \
+ apt-transport-https \
+ build-essential \
+ curl \
+ wget \
+ git \
+ bzip2 \
+ ca-certificates \
+ sudo \
+ locales \
+ fonts-liberation \
+ run-one && \
+ apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN echo "$LOG_TAG Set locale" && \
+ echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \
+ echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \
+ echo "LANG=en_US.UTF-8" > /etc/locale.conf && \
+ locale-gen en_US.UTF-8
+
+ENV NB_USER=$NB_USER \
+ NB_UID=$NB_UID \
+ NB_PREFIX=$NB_PREFIX \
+ NB_PORT=$NB_PORT \
+ CONDA_DIR=/opt/conda \
+ LANG=en_US.UTF-8 \
+ LC_ALL=en_US.UTF-8 \
+ LANGUAGE=en_US.UTF-8
+ENV PATH=$CONDA_DIR/bin:$PATH \
+ HOME=/home/$NB_USER
+
+ENV MLFLOW_TRACKING_URI=$MLFLOW_TRACKING_URI
+
+# Create NB_USER user with UID=1000 and in the 'users' group
+RUN useradd -M -s /bin/bash -N -u $NB_UID $NB_USER && \
+ chown -R ${NB_USER}:users /usr/local/bin && \
+ mkdir -p $HOME && \
+ chown -R ${NB_USER}:users ${HOME} && \
+ mkdir -p ${CONDA_DIR} && \
+ chown -R ${NB_USER}:users ${CONDA_DIR} && \
+ chmod g+w /etc/passwd
+
+# Add Tini
+ENV TINI_VERSION v0.19.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini
/tini
+RUN mv /tini /usr/local/bin/tini && chmod +x /usr/local/bin/tini
+
+# Install conda
+USER $NB_UID
+ARG PYTHON_VERSION=default
+ENV MINICONDA_VERSION=4.8.3 \
+ MINICONDA_MD5=751786b92c00b1aeae3f017b781018df \
+ CONDA_VERSION=4.8.3
+
+WORKDIR /tmp
+RUN wget --quiet
https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh
&& \
+ echo "${MINICONDA_MD5}
*Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
+ /bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p
$CONDA_DIR && \
+ rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+ echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \
+ conda config --system --prepend channels conda-forge && \
+ conda config --system --set auto_update_conda false && \
+ conda config --system --set show_channel_urls true && \
+ conda config --system --set channel_priority strict && \
+ if [ ! $PYTHON_VERSION = 'default' ]; then conda install --yes
python=$PYTHON_VERSION; fi && \
+ conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed
's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \
+ conda clean --all -f -y && \
+ rm -rf /home/$NB_USER/.cache/yarn
+
+# Install latest sumbarine python sdk and notebook
+RUN git clone https://github.com/apache/submarine && \
+ pip install submarine/submarine-sdk/pysubmarine && \
+ conda install nodejs && \
+ conda install -c conda-forge jupyterlab jupyterlab-git && \
+ jupyter lab build
+
+# Add DeepFM example into notebook
+RUN cp
submarine/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb
$HOME && \
+ cp -r
submarine/submarine-sdk/pysubmarine/example/{data,deepfm_example.ipynb,deepfm.json}
$HOME && \
+ rm submarine -rf
+
+# Install latest stable qlib
+RUN pip install numpy==1.19.5 pyqlib==0.6.1
+
+# Add qlib example in notebook
+RUN wget
https://raw.githubusercontent.com/microsoft/qlib/main/examples/workflow_by_code.ipynb
-P $HOME
+
+
+EXPOSE $NB_PORT
+ENTRYPOINT ["tini", "-g", "--"]
+CMD ["start-notebook.sh"]
+COPY --chown=${NB_USER}:users start-notebook.sh /usr/local/bin
+WORKDIR ${HOME}
diff --git a/dev-support/docker-images/jupyter-gpu/build.sh
b/dev-support/docker-images/jupyter-gpu/build.sh
new file mode 100755
index 0000000..0349e45
--- /dev/null
+++ b/dev-support/docker-images/jupyter-gpu/build.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euxo pipefail
+
+JUPYTER_IMAGE="apache/submarine:jupyter-notebook-gpu-0.6.0-SNAPSHOT"
+
+if [ -L ${BASH_SOURCE-$0} ]; then
+ PWD=$(dirname $(readlink "${BASH_SOURCE-$0}"))
+else
+ PWD=$(dirname ${BASH_SOURCE-$0})
+fi
+export CURRENT_PATH=$(cd "${PWD}">/dev/null; pwd)
+SUBMARINE_HOME=${CURRENT_PATH}/../../..
+
+# build image
+echo "Start building the ${JUPYTER_IMAGE} docker image ..."
+cd ${CURRENT_PATH}
+docker build -t ${JUPYTER_IMAGE} .
diff --git a/dev-support/docker-images/jupyter-gpu/start-notebook.sh
b/dev-support/docker-images/jupyter-gpu/start-notebook.sh
new file mode 100755
index 0000000..fce5e33
--- /dev/null
+++ b/dev-support/docker-images/jupyter-gpu/start-notebook.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# description: Start and stop daemon script for.
+#
+
+set -euo pipefail
+
+# Install conda dependency
+if [[ -n "${INSTALL_ENVIRONMENT_COMMAND:-}" ]]; then
+ /bin/bash -c "${INSTALL_ENVIRONMENT_COMMAND}"
+fi
+
+NOTEBOOK_ARGS="--ip=0.0.0.0 --no-browser --allow-root --NotebookApp.token=''
--NotebookApp.password='' --NotebookApp.allow_origin='*'"
+NB_USER="${NB_USER:-"jovyan"}"
+NB_PREFIX="${NB_PREFIX:-"/"}"
+NB_PORT="${NB_PORT:-8888}"
+
+if [[ -n "${NB_USER}" ]]; then
+ NOTEBOOK_ARGS="--notebook-dir=/home/${NB_USER} ${NOTEBOOK_ARGS}"
+fi
+
+if [[ -n "${NB_PORT}" ]]; then
+ NOTEBOOK_ARGS="--port=${NB_PORT} ${NOTEBOOK_ARGS}"
+fi
+
+if [[ -n "${NB_PREFIX}" ]]; then
+ NOTEBOOK_ARGS="--NotebookApp.base_url=${NB_PREFIX} ${NOTEBOOK_ARGS}"
+fi
+
+/bin/bash -c "jupyter notebook ${NOTEBOOK_ARGS}"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
