This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new d43c5cd SUBMARINE-560. Build an API to manage notebook instances with
Swagger
d43c5cd is described below
commit d43c5cd6244082f1ff2d90e6cca48ccb37d1214f
Author: Ryan Lo <lowc1...@gmail.com>
AuthorDate: Fri Jul 31 17:12:01 2020 +0800
SUBMARINE-560. Build an API to manage notebook instances with Swagger
### What is this PR for?
User can create his or her notebook instances in specified namespace.
### What type of PR is it?
[Feature]
### Todos
Will open another PR to finish following todos.
1. Add test for Notebook Rest API
2. User can list/delete notebook instances via REST API
### What is the Jira issue?
[SUBMARINE-560](https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-560)
### How should this be tested?
[Travis
CI](https://travis-ci.org/github/lowc1012/submarine/builds/711003012)
### Screenshots (if appropriate)
<img width="697" alt="image1"
src="https://user-images.githubusercontent.com/52355146/88690339-248ad300-d12e-11ea-9cb8-9c9e7f4a5b1c.png";>
<img width="569" alt="image2"
src="https://user-images.githubusercontent.com/52355146/88690409-39676680-d12e-11ea-8102-b3d92dd99c80.png";>
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Ryan Lo <lowc1...@gmail.com>
Closes #353 from lowc1012/SUBMARINE-560 and squashes the following commits:
faf5026 [Ryan Lo] SUBMARINE-560. resolve conflict & change JUPYTER_IMAGE
785bc64 [Ryan Lo] SUBMARINE-560. fix bug & add "start-notebook.sh" in
images for jupyter container
ca77284 [Ryan Lo] SUBMARINE-560. Update K8sSubmitter.java
ae08107 [Ryan Lo] SUBMARINE-560. WIP. Build an REST API to manage notebook
instances
---
dev-support/docker-images/jupyter/Dockerfile | 53 ++++--
dev-support/docker-images/jupyter/build.sh | 6 +-
.../docker-images/jupyter/start-notebook.sh | 45 +++++
.../server/api/{experiment => }/Submitter.java | 30 +++-
.../submarine/server/api/notebook/Notebook.java | 120 +++++++++++++
.../submarine/server/api/notebook/NotebookId.java | 69 ++++++++
.../submarine/server/api/spec/NotebookMeta.java | 61 +++++++
.../submarine/server/api/spec/NotebookPodSpec.java | 119 +++++++++++++
.../submarine/server/api/spec/NotebookSpec.java} | 62 ++++---
.../apache/submarine/server/SubmitterManager.java | 2 +-
.../server/experiment/ExperimentManager.java | 2 +-
.../NotebookIdDeserializer.java} | 36 ++--
.../NotebookIdSerializer.java} | 35 ++--
.../submarine/server/notebook/NotebookManager.java | 139 +++++++++++++++
.../submarine/server/response/JsonResponse.java | 5 +
.../submarine/server/rest/NotebookRestApi.java | 172 ++++++++++++++++++
.../submarine/server/rest/RestConstants.java | 12 +-
.../server/submitter/k8s/K8sSubmitter.java | 62 ++++++-
.../server/submitter/k8s/model/NotebookCR.java | 115 ++++++++++++
.../server/submitter/k8s/model/NotebookCRSpec.java | 92 ++++++++++
.../submitter/k8s/parser/NotebookSpecParser.java | 194 +++++++++++++++++++++
.../submitter/k8s/ExperimentSpecParserTest.java | 35 ++--
.../server/submitter/k8s/K8SJobSubmitterTest.java | 4 +-
.../server/submitter/k8s/MLJobConverterTest.java | 4 +-
.../submitter/k8s/NotebookSpecParserTest.java | 76 ++++++++
.../server/submitter/k8s/SpecBuilder.java | 10 +-
.../src/test/resources/notebook_req.json | 15 ++
27 files changed, 1457 insertions(+), 118 deletions(-)
diff --git a/dev-support/docker-images/jupyter/Dockerfile
b/dev-support/docker-images/jupyter/Dockerfile
index 110ed43..f513a6d 100644
--- a/dev-support/docker-images/jupyter/Dockerfile
+++ b/dev-support/docker-images/jupyter/Dockerfile
@@ -13,9 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-ARG BASE_IMAGE=tensorflow/tensorflow:2.1.0-py3-jupyter
-
-FROM $BASE_IMAGE
+FROM ubuntu:18.04
ARG NB_USER="jovyan"
ARG NB_UID="1000"
@@ -28,8 +26,8 @@ ENV NB_UID $NB_UID
ENV NB_GID $NB_GID
ENV NB_PREFIX $NB_PREFIX
ENV NB_PORT $NB_PORT
-
-ENV PATH=$HOME/.local/bin:$PATH
+ENV CONDA_DIR=/opt/conda
+ENV PATH=$CONDA_DIR/bin:$PATH
ENV HOME=/home/$NB_USER
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq
--no-install-recommends \
@@ -43,10 +41,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get
install -yq --no-in
sudo \
locales \
fonts-liberation \
- run-one \
- python3-pip \
- python3-dev \
- python3-setuptools && \
+ run-one && \
apt-get clean && rm -rf /var/lib/apt/lists/*
RUN echo "$LOG_TAG Set locale" && \
@@ -70,12 +65,40 @@ ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini
/tini
RUN mv /tini /usr/local/bin/tini && chmod +x /usr/local/bin/tini
-# Install python package
-RUN pip uninstall -y enum34
-RUN pip3 --no-cache-dir install jupyterlab
+# Install conda
+ARG PYTHON_VERSION=default
+ENV MINICONDA_VERSION=4.8.3 \
+ MINICONDA_MD5=751786b92c00b1aeae3f017b781018df \
+ CONDA_VERSION=4.8.3
+
+WORKDIR /tmp
+RUN wget --quiet
https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh
&& \
+ echo "${MINICONDA_MD5}
*Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
+ /bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p
$CONDA_DIR && \
+ rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+ echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \
+ conda config --system --prepend channels conda-forge && \
+ conda config --system --set auto_update_conda false && \
+ conda config --system --set show_channel_urls true && \
+ conda config --system --set channel_priority strict && \
+ if [ ! $PYTHON_VERSION = 'default' ]; then conda install --yes
python=$PYTHON_VERSION; fi && \
+ conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed
's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \
+ conda install --quiet --yes conda && \
+ conda install --quiet --yes pip && \
+ conda clean --all -f -y && \
+ rm -rf /home/$NB_USER/.cache/yarn
+
+# Install jupyter
+RUN conda install --quiet --yes notebook=6.0.3 && \
+ conda clean --all -f -y
+
+USER root
+RUN mkdir -p $CONDA_DIR && \
+ chown -R ${NB_USER}:users $CONDA_DIR
-# Configure container startup
+USER $NB_USER
EXPOSE $NB_PORT
-USER $NB_UID
ENTRYPOINT ["tini", "-g", "--"]
-CMD ["sh","-c", "jupyter notebook --notebook-dir=/home/${NB_USER} --ip=0.0.0.0
--no-browser --allow-root --port=${NB_PORT} --NotebookApp.token=''
--NotebookApp.password='' --NotebookApp.allow_origin='*'
--NotebookApp.base_url=${NB_PREFIX}"]
+WORKDIR ${HOME}
+CMD ["start-notebook.sh"]
+COPY start-notebook.sh /usr/local/bin
diff --git a/dev-support/docker-images/jupyter/build.sh
b/dev-support/docker-images/jupyter/build.sh
index 200f0eb..977a6cc 100755
--- a/dev-support/docker-images/jupyter/build.sh
+++ b/dev-support/docker-images/jupyter/build.sh
@@ -16,7 +16,7 @@
set -euxo pipefail
-TF_JUPYTER_IMAGE="apache/submarine:tf2.1.0-jupyter"
+JUPYTER_IMAGE="apache/submarine:jupyter-notebook-0.5.0-SNAPSHOT"
if [ -L ${BASH_SOURCE-$0} ]; then
PWD=$(dirname $(readlink "${BASH_SOURCE-$0}"))
@@ -27,6 +27,6 @@ export CURRENT_PATH=$(cd "${PWD}">/dev/null; pwd)
SUBMARINE_HOME=${CURRENT_PATH}/../../..
# build image
-echo "Start building the ${TF_JUPYTER_IMAGE} docker image ..."
+echo "Start building the ${JUPYTER_IMAGE} docker image ..."
cd ${CURRENT_PATH}
-docker build -t ${TF_JUPYTER_IMAGE} .
+docker build -t ${JUPYTER_IMAGE} .
diff --git a/dev-support/docker-images/jupyter/start-notebook.sh
b/dev-support/docker-images/jupyter/start-notebook.sh
new file mode 100755
index 0000000..e39def5
--- /dev/null
+++ b/dev-support/docker-images/jupyter/start-notebook.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# description: Start and stop daemon script for.
+#
+
+set -euo pipefail
+
+if [[ -n "${ENVIRONMENT_COMMAND:-}" ]]; then
+ /bin/bash -c "${ENVIRONMENT_COMMAND}"
+fi
+
+NOTEBOOK_ARGS="--ip=0.0.0.0 --no-browser --allow-root --NotebookApp.token=''
--NotebookApp.password='' --NotebookApp.allow_origin='*'"
+NB_USER="${NB_USER:-"jovyan"}"
+NB_PREFIX="${NB_PREFIX:-"/"}"
+NB_PORT="${NB_PORT:-8888}"
+
+if [[ -n "${NB_USER}" ]]; then
+ NOTEBOOK_ARGS="--notebook-dir=/home/${NB_USER} ${NOTEBOOK_ARGS}"
+fi
+
+if [[ -n "${NB_PORT}" ]]; then
+ NOTEBOOK_ARGS="--port=${NB_PORT} ${NOTEBOOK_ARGS}"
+fi
+
+if [[ -n "${NB_PREFIX}" ]]; then
+ NOTEBOOK_ARGS="--NotebookApp.base_url=${NB_PREFIX} ${NOTEBOOK_ARGS}"
+fi
+
+/bin/bash -c "jupyter notebook ${NOTEBOOK_ARGS}"
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/Submitter.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
similarity index 73%
rename from
submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/Submitter.java
rename to
submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
index 401fb01..8c7ea49 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/Submitter.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
@@ -17,11 +17,15 @@
* under the License.
*/
-package org.apache.submarine.server.api.experiment;
+package org.apache.submarine.server.api;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
+import org.apache.submarine.server.api.experiment.Experiment;
+import org.apache.submarine.server.api.experiment.ExperimentLog;
+import org.apache.submarine.server.api.notebook.Notebook;
import org.apache.submarine.server.api.spec.ExperimentSpec;
+import org.apache.submarine.server.api.spec.NotebookSpec;
/**
* The submitter should implement this interface.
@@ -81,4 +85,28 @@ public interface Submitter {
* @throws SubmarineRuntimeException running error
*/
ExperimentLog getExperimentLogName(ExperimentSpec spec, String id) throws
SubmarineRuntimeException;
+
+ /**
+ * Create a notebook with spec
+ * @param spec notebook spec
+ * @return object
+ * @throws SubmarineRuntimeException running error
+ */
+ Notebook createNotebook(NotebookSpec spec) throws SubmarineRuntimeException;
+
+ /**
+ * Find a notebook with spec
+ * @param spec spec
+ * @return object
+ * @throws SubmarineRuntimeException running error
+ */
+ Notebook findNotebook(NotebookSpec spec) throws SubmarineRuntimeException;
+
+ /**
+ * Delete a notebook with spec
+ * @param spec spec
+ * @return object
+ * @throws SubmarineRuntimeException running error
+ */
+ Notebook deleteNotebook(NotebookSpec spec) throws SubmarineRuntimeException;
}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/Notebook.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/Notebook.java
new file mode 100644
index 0000000..b71aded
--- /dev/null
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/Notebook.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.api.notebook;
+
+import org.apache.submarine.server.api.spec.NotebookSpec;
+
+/**
+ * The notebook instance in submarine
+ */
+public class Notebook {
+ private NotebookId notebookId;
+ private String name;
+ private String uid;
+ private String url;
+ private String status;
+ private String createdTime;
+ private String deletedTime;
+ private NotebookSpec spec;
+
+ public NotebookId getNotebookId() {
+ return notebookId;
+ }
+
+ public void setNotebookId(NotebookId notebookId) {
+ this.notebookId = notebookId;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getUid() {
+ return uid;
+ }
+
+ public void setUid(String uid) {
+ this.uid = uid;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ public void setStatus(String status) {
+ this.status = status;
+ }
+
+ public String getCreatedTime() {
+ return createdTime;
+ }
+
+ public void setCreatedTime(String createdTime) {
+ this.createdTime = createdTime;
+ }
+
+ public String getDeletedTime() {
+ return deletedTime;
+ }
+
+ public void setDeletedTime(String deletedTime) {
+ this.deletedTime = deletedTime;
+ }
+
+ public NotebookSpec getSpec() {
+ return spec;
+ }
+
+ public void setSpec(NotebookSpec spec) {
+ this.spec = spec;
+ }
+
+ public enum Status {
+ STATUS_CREATED("Created"),
+ STATUS_DELETED("Deleted");
+
+ private String value;
+ Status(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
+ }
+
+}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/NotebookId.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/NotebookId.java
new file mode 100644
index 0000000..34f0a10
--- /dev/null
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/NotebookId.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.api.notebook;
+
+import org.apache.submarine.commons.utils.AbstractUniqueIdGenerator;
+
+/**
+ * The unique id for notebook. Formatter:
+ * notebook_${server_timestamp}_${counter} Such as:
+ * notebook_1577627710_0001
+ */
+public class NotebookId extends AbstractUniqueIdGenerator<NotebookId> {
+ private static final String NOTEBOOK_ID_PREFIX = "notebook_";
+
+ /**
+ * Get the object of NotebookId.
+ * @param notebookId string
+ * @return object
+ */
+ public static NotebookId fromString(String notebookId) {
+ if (notebookId == null) {
+ return null;
+ }
+ String[] components = notebookId.split("\\_");
+ if (components.length != 3) {
+ return null;
+ }
+ return NotebookId.newInstance(Long.parseLong(components[1]),
Integer.parseInt(components[2]));
+ }
+
+ /**
+ * Get the object of NotebookId.
+ * @param serverTimestamp timestamp
+ * @param id count
+ * @return object
+ */
+ public static NotebookId newInstance(long serverTimestamp, int id) {
+ NotebookId notebookId = new NotebookId();
+ notebookId.setServerTimestamp(serverTimestamp);
+ notebookId.setId(id);
+ return notebookId;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(64);
+ sb.append(NOTEBOOK_ID_PREFIX).append(getServerTimestamp()).append("_");
+ format(sb, getId());
+ return sb.toString();
+ }
+
+}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookMeta.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookMeta.java
new file mode 100644
index 0000000..b8d314c
--- /dev/null
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookMeta.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.api.spec;
+
+public class NotebookMeta {
+ private String name;
+ private String namespace;
+
+ public NotebookMeta() {
+
+ }
+
+ /**
+ * Get the notebook name which is unique within a namespace.
+ * @return notebook name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Name must be unique within a namespace. Is required when creating
notebook.
+ * @param name notebook name
+ */
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Get the namespace which defines the isolated space for each notebook.
+ * @return namespace
+ */
+ public String getNamespace() {
+ return namespace;
+ }
+
+ /**
+ * Namespace defines the space within each name must be unique.
+ * @param namespace namespace
+ */
+ public void setNamespace(String namespace) {
+ this.namespace = namespace;
+ }
+}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookPodSpec.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookPodSpec.java
new file mode 100644
index 0000000..64e78db
--- /dev/null
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookPodSpec.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.api.spec;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class NotebookPodSpec {
+
+ private Map<String, String> envVars;
+ private String resources;
+
+ // should ignored in JSON Serialization
+ private transient Map<String, String> resourceMap;
+
+ public NotebookPodSpec() {
+
+ }
+
+ /**
+ * Get envVars
+ * @return envVars
+ */
+ public Map<String, String> getEnvVars() {
+ return envVars;
+ }
+
+ /**
+ * Set envVars
+ * @param envVars environment variables for container
+ */
+ public void setEnvVars(Map<String, String> envVars) {
+ this.envVars = envVars;
+ }
+
+ /**
+ * Get the resources for container.
+ * Resource type
+ * cpu: vCPU/Core
+ * memory: E/Ei, P/Pi, T/Ti, G/Gi, M/Mi, K/Ki
+ * nvidia.com/gpu: GPUs (not possible to request a fraction of a GPU)
+ * such as: cpu=1,memory=2Gi,nvidia.com/gpu=1
+ * @return resources resources for container
+ */
+ public String getResources() {
+ return resources;
+ }
+
+ /**
+ * Set the resource for container
+ * Resource type
+ * cpu: vCPU/Core
+ * memory: E/Ei, P/Pi, T/Ti, G/Gi, M/Mi, K/Ki
+ * nvidia.com/gpu: GPUs (not possible to request a fraction of a GPU)
+ * such as: cpu=1,memory=2Gi,nvidia.com/gpu=1
+ * @param resources resources for container
+ */
+ public void setResources(String resources) {
+ this.resources = resources;
+ parseResources();
+ }
+
+ /**
+ * Parse resources
+ */
+ public void parseResources() {
+ if (resources != null) {
+ resourceMap = new HashMap<>();
+ for (String item : resources.split(",")) {
+ String[] r = item.split("=");
+ if (r.length == 2) {
+ resourceMap.put(r[0], r[1]);
+ }
+ }
+ }
+ }
+
+ /**
+ * Get the cpu reserved by the notebook server
+ * @return String or null
+ */
+ public String getCpu() {
+ return resourceMap.get("cpu");
+ }
+
+ /**
+ * Get the memory reserved by the notebook server
+ * @return String or null
+ */
+ public String getMemory() {
+ return resourceMap.get("memory");
+ }
+
+ /**
+ * Get the gpu reserved by the notebook server
+ * @return String or null
+ */
+ public String getGpu() {
+ return resourceMap.get("nvidia.com/gpu");
+ }
+
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookSpec.java
similarity index 50%
copy from
submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
copy to
submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookSpec.java
index f7be478..3a5a5f6 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookSpec.java
@@ -17,30 +17,40 @@
* under the License.
*/
-package org.apache.submarine.server.rest;
-
-public class RestConstants {
- public static final String V1 = "v1";
- public static final String EXPERIMENT = "experiment";
- public static final String ID = "id";
- public static final String PING = "ping";
- public static final String MEDIA_TYPE_YAML = "application/yaml";
- public static final String CHARSET_UTF8 = "charset=utf-8";
-
- public static final String METASTORE = "metastore";
-
- public static final String CLUSTER = "cluster";
- public static final String ADDRESS = "address";
-
- public static final String NODES = "nodes";
- public static final String NODE = "node";
-
- public static final String LOGS = "logs";
-
- /**
- * Environment
- */
- public static final String ENVIRONMENT = "environment";
-
- public static final String ENVIRONMENT_ID = "id";
+package org.apache.submarine.server.api.spec;
+
+public class NotebookSpec {
+
+ private NotebookMeta meta;
+ private EnvironmentSpec environment;
+ private NotebookPodSpec spec;
+
+ public NotebookSpec() {
+
+ }
+
+ public NotebookMeta getMeta() {
+ return meta;
+ }
+
+ public void setMeta(NotebookMeta meta) {
+ this.meta = meta;
+ }
+
+ public EnvironmentSpec getEnvironment() {
+ return environment;
+ }
+
+ public void setEnvironment(EnvironmentSpec environmentSpec) {
+ this.environment = environmentSpec;
+ }
+
+ public NotebookPodSpec getSpec() {
+ return spec;
+ }
+
+ public void setSpec(NotebookPodSpec podSpec) {
+ this.spec = podSpec;
+ }
+
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
index 7748b3e..f2577bf 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
@@ -21,7 +21,7 @@ package org.apache.submarine.server;
import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
-import org.apache.submarine.server.api.experiment.Submitter;
+import org.apache.submarine.server.api.Submitter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
index 93b7947..346f77c 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
@@ -33,7 +33,7 @@ import org.apache.submarine.server.SubmarineServer;
import org.apache.submarine.server.SubmitterManager;
import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.ExperimentId;
-import org.apache.submarine.server.api.experiment.Submitter;
+import org.apache.submarine.server.api.Submitter;
import org.apache.submarine.server.api.experiment.ExperimentLog;
import org.apache.submarine.server.api.spec.ExperimentSpec;
import org.slf4j.Logger;
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/gson/NotebookIdDeserializer.java
similarity index 50%
copy from
submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
copy to
submarine-server/server-core/src/main/java/org/apache/submarine/server/gson/NotebookIdDeserializer.java
index f7be478..df3a8c9 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/gson/NotebookIdDeserializer.java
@@ -17,30 +17,20 @@
* under the License.
*/
-package org.apache.submarine.server.rest;
+package org.apache.submarine.server.gson;
-public class RestConstants {
- public static final String V1 = "v1";
- public static final String EXPERIMENT = "experiment";
- public static final String ID = "id";
- public static final String PING = "ping";
- public static final String MEDIA_TYPE_YAML = "application/yaml";
- public static final String CHARSET_UTF8 = "charset=utf-8";
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
+import org.apache.submarine.server.api.notebook.NotebookId;
- public static final String METASTORE = "metastore";
+import java.lang.reflect.Type;
- public static final String CLUSTER = "cluster";
- public static final String ADDRESS = "address";
-
- public static final String NODES = "nodes";
- public static final String NODE = "node";
-
- public static final String LOGS = "logs";
-
- /**
- * Environment
- */
- public static final String ENVIRONMENT = "environment";
-
- public static final String ENVIRONMENT_ID = "id";
+public class NotebookIdDeserializer implements JsonDeserializer<NotebookId> {
+ @Override
+ public NotebookId deserialize(JsonElement json, Type typeOfT,
JsonDeserializationContext context)
+ throws JsonParseException {
+ return NotebookId.fromString(json.getAsJsonPrimitive().getAsString());
+ }
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/gson/NotebookIdSerializer.java
similarity index 50%
copy from
submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
copy to
submarine-server/server-core/src/main/java/org/apache/submarine/server/gson/NotebookIdSerializer.java
index f7be478..f5baeb6 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/gson/NotebookIdSerializer.java
@@ -17,30 +17,19 @@
* under the License.
*/
-package org.apache.submarine.server.rest;
+package org.apache.submarine.server.gson;
-public class RestConstants {
- public static final String V1 = "v1";
- public static final String EXPERIMENT = "experiment";
- public static final String ID = "id";
- public static final String PING = "ping";
- public static final String MEDIA_TYPE_YAML = "application/yaml";
- public static final String CHARSET_UTF8 = "charset=utf-8";
+import com.google.gson.JsonElement;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+import org.apache.submarine.server.api.notebook.NotebookId;
- public static final String METASTORE = "metastore";
+import java.lang.reflect.Type;
- public static final String CLUSTER = "cluster";
- public static final String ADDRESS = "address";
-
- public static final String NODES = "nodes";
- public static final String NODE = "node";
-
- public static final String LOGS = "logs";
-
- /**
- * Environment
- */
- public static final String ENVIRONMENT = "environment";
-
- public static final String ENVIRONMENT_ID = "id";
+public class NotebookIdSerializer implements JsonSerializer<NotebookId> {
+ @Override
+ public JsonElement serialize(NotebookId src, Type typeOfSrc,
JsonSerializationContext context) {
+ return new JsonPrimitive(src.toString());
+ }
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
new file mode 100644
index 0000000..5547c23
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.notebook;
+
+import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
+import org.apache.submarine.server.SubmarineServer;
+import org.apache.submarine.server.SubmitterManager;
+import org.apache.submarine.server.api.Submitter;
+import org.apache.submarine.server.api.notebook.Notebook;
+import org.apache.submarine.server.api.notebook.NotebookId;
+import org.apache.submarine.server.api.spec.NotebookSpec;
+
+import javax.ws.rs.core.Response;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class NotebookManager {
+
+ private static volatile NotebookManager manager;
+
+ private final Submitter submitter;
+
+ private NotebookManager(Submitter submitter) {
+ this.submitter = submitter;
+ }
+
+ private final AtomicInteger notebookCounter = new AtomicInteger(0);
+
+ /**
+ * Used to cache the specs by the notebook id.
+ * key: the string of notebook id
+ * value: Notebook object
+ */
+ private final ConcurrentMap<String, Notebook> cachedNotebookMap = new
ConcurrentHashMap<>();
+
+ /**
+ * Get the singleton instance
+ * @return object
+ */
+ public static NotebookManager getInstance() {
+ if (manager == null) {
+ synchronized (NotebookManager.class) {
+ if (manager == null) {
+ manager = new NotebookManager(SubmitterManager.loadSubmitter());
+ }
+ }
+ }
+ return manager;
+ }
+
+ /**
+ * Create a notebook instance
+ * @param spec NotebookSpec
+ * @return object
+ * @throws SubmarineRuntimeException the service error
+ */
+ public Notebook createNotebook(NotebookSpec spec) throws
SubmarineRuntimeException {
+ checkNotebookSpec(spec);
+ Notebook notebook = submitter.createNotebook(spec);
+ notebook.setNotebookId(generateNotebookId());
+ notebook.setSpec(spec);
+ cachedNotebookMap.putIfAbsent(notebook.getNotebookId().toString(),
notebook);
+ return notebook;
+ }
+
+ /**
+ * List notebook instances
+ * @param status status, if null will return all notebooks
+ * @return list
+ * @throws SubmarineRuntimeException the service error
+ */
+ public List<Notebook> listNotebooksByStatus(String status) throws
SubmarineRuntimeException {
+ //TODO(ryan): implement the method
+ return null;
+ }
+
+ /**
+ * Get a notebook instance
+ * @param id notebook id
+ * @return object
+ * @throws SubmarineRuntimeException the service error
+ */
+ public Notebook getNotebook(String id) throws SubmarineRuntimeException {
+ //TODO(ryan): implement the method
+ return null;
+ }
+
+ /**
+ * Delete the notebook instance
+ * @param id notebook id
+ * @return object
+ * @throws SubmarineRuntimeException the service error
+ */
+ public Notebook deleteNotebook(String id) throws SubmarineRuntimeException {
+ //TODO(ryan): implement the method
+ return null;
+ }
+
+ /**
+ * Generate a unique notebook id
+ * @return notebook id
+ */
+ private NotebookId generateNotebookId() {
+ return NotebookId.newInstance(SubmarineServer.getServerTimeStamp(),
+ notebookCounter.incrementAndGet());
+ }
+
+ /**
+ * Check if notebook spec is valid
+ * @param spec notebook spec
+ */
+ private void checkNotebookSpec(NotebookSpec spec) {
+ //TODO(ryan): The method need to be improved
+ if (spec == null) {
+ throw new SubmarineRuntimeException(Response.Status.OK.getStatusCode(),
+ "Invalid. Notebook Spec object is null.");
+ }
+ }
+
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/JsonResponse.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/JsonResponse.java
index b4a4e19..27d4cd1 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/JsonResponse.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/JsonResponse.java
@@ -24,8 +24,11 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.TypeAdapter;
import org.apache.submarine.server.api.experiment.ExperimentId;
+import org.apache.submarine.server.api.notebook.NotebookId;
import org.apache.submarine.server.gson.ExperimentIdDeserializer;
import org.apache.submarine.server.gson.ExperimentIdSerializer;
+import org.apache.submarine.server.gson.NotebookIdDeserializer;
+import org.apache.submarine.server.gson.NotebookIdSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -171,6 +174,8 @@ public class JsonResponse<T> {
.registerTypeAdapter(Date.class, safeDateTypeAdapter)
.registerTypeAdapter(ExperimentId.class, new
ExperimentIdSerializer())
.registerTypeAdapter(ExperimentId.class, new
ExperimentIdDeserializer())
+ .registerTypeAdapter(NotebookId.class, new NotebookIdSerializer())
+ .registerTypeAdapter(NotebookId.class, new NotebookIdDeserializer())
.serializeNulls()
.create();
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/NotebookRestApi.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/NotebookRestApi.java
new file mode 100644
index 0000000..ac61cb2
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/NotebookRestApi.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.rest;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+
+import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
+import org.apache.submarine.server.api.notebook.Notebook;
+import org.apache.submarine.server.api.spec.NotebookSpec;
+import org.apache.submarine.server.notebook.NotebookManager;
+import org.apache.submarine.server.response.JsonResponse;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.List;
+
+/**
+ * Notebook Service REST API v1
+ */
+@Path(RestConstants.V1 + "/" + RestConstants.NOTEBOOK)
+@Produces({MediaType.APPLICATION_JSON + "; " + RestConstants.CHARSET_UTF8})
+public class NotebookRestApi {
+
+ /* Notebook manager */
+ private final NotebookManager notebookManager =
NotebookManager.getInstance();
+
+ /**
+ * Return the Pong message for test the connectivity
+ * @return Pong message
+ */
+ @GET
+ @Path(RestConstants.PING)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Operation(summary = "Ping submarine server",
+ tags = {"notebook"},
+ description = "Return the Pong message for test the connectivity",
+ responses = {
+ @ApiResponse(responseCode = "200", description = "successful
operation",
+ content = @Content(schema = @Schema(implementation =
String.class)))})
+ public Response ping() {
+ return new JsonResponse.Builder<String>(Response.Status.OK)
+ .success(true).result("Pong").build();
+ }
+
+ /**
+ * Create a notebook with spec
+ * @param spec notebook spec
+ * @return the detailed info about created notebook
+ */
+ @POST
+ @Consumes({RestConstants.MEDIA_TYPE_YAML, MediaType.APPLICATION_JSON})
+ @Operation(
+ summary = "Create a notebook instance",
+ tags = {"notebook"},
+ responses = {
+ @ApiResponse(description = "successful operation", content =
@Content(
+ schema = @Schema(implementation =
JsonResponse.class)))})
+ public Response createNotebook(NotebookSpec spec) {
+ try {
+ Notebook notebook = notebookManager.createNotebook(spec);
+ return new
JsonResponse.Builder<Notebook>(Response.Status.OK).success(true)
+ .message("Create a notebook instance").result(notebook).build();
+ } catch (SubmarineRuntimeException e) {
+ return parseNotebookServiceException(e);
+ }
+ }
+
+ /**
+ * List all notebooks created by the user
+ * @param status status
+ * @return notebook list
+ */
+ @GET
+ @Operation(
+ summary = "List notebooks",
+ tags = {"notebook"},
+ responses = {
+ @ApiResponse(description = "successful operation", content =
@Content(
+ schema = @Schema(implementation =
JsonResponse.class)))})
+ public Response listNotebooks(@QueryParam("status") String status) {
+ try {
+ List<Notebook> notebookList =
notebookManager.listNotebooksByStatus(status);
+ return new
JsonResponse.Builder<List<Notebook>>(Response.Status.OK).success(true)
+ .result(notebookList).build();
+ } catch (SubmarineRuntimeException e) {
+ return parseNotebookServiceException(e);
+ }
+ }
+
+ /**
+ * Get detailed info about the notebook by notebook id
+ * @param id notebook id
+ * @return detailed info about the notebook
+ */
+ @GET
+ @Path("/{id}")
+ @Operation(
+ summary = "Get detailed info about the notebook",
+ tags = {"notebook"},
+ responses = {
+ @ApiResponse(
+ description = "successful operation", content =
@Content(
+ schema = @Schema(implementation =
JsonResponse.class))),
+ @ApiResponse(responseCode = "404", description = "Notebook
not found")})
+ public Response getNotebook(@PathParam(RestConstants.NOTEBOOK_ID) String id)
{
+ try {
+ Notebook notebook = notebookManager.getNotebook(id);
+ return new
JsonResponse.Builder<Notebook>(Response.Status.OK).success(true)
+ .result(notebook).build();
+ } catch (SubmarineRuntimeException e) {
+ return parseNotebookServiceException(e);
+ }
+ }
+
+ /**
+ * Delete the notebook with notebook id
+ * @param id notebook id
+ * @return the detailed info about deleted notebook
+ */
+ @DELETE
+ @Path("/{id}")
+ @Operation(
+ summary = "Delete the notebook",
+ tags = {"notebook"},
+ responses = {
+ @ApiResponse(
+ description = "successful operation", content =
@Content(
+ schema = @Schema(implementation =
JsonResponse.class))),
+ @ApiResponse(responseCode = "404", description = "Notebook
not found")})
+ public Response deleteNotebook(@PathParam(RestConstants.NOTEBOOK_ID) String
id) {
+ try {
+ Notebook notebook = notebookManager.deleteNotebook(id);
+ return new
JsonResponse.Builder<Notebook>(Response.Status.OK).success(true)
+ .result(notebook).build();
+ } catch (SubmarineRuntimeException e) {
+ return parseNotebookServiceException(e);
+ }
+ }
+
+ private Response parseNotebookServiceException(SubmarineRuntimeException e) {
+ return new
JsonResponse.Builder<String>(e.getCode()).message(e.getMessage()).build();
+ }
+
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
index f7be478..ecfce40 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/RestConstants.java
@@ -36,11 +36,19 @@ public class RestConstants {
public static final String NODE = "node";
public static final String LOGS = "logs";
-
+
/**
* Environment
*/
public static final String ENVIRONMENT = "environment";
-
+
public static final String ENVIRONMENT_ID = "id";
+
+ /**
+ * Notebook
+ */
+ public static final String NOTEBOOK = "notebook";
+
+ public static final String NOTEBOOK_ID = "id";
+
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
index 9b495bf..5fe17ad 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
@@ -38,14 +38,19 @@ import io.kubernetes.client.util.KubeConfig;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.server.api.exception.InvalidSpecException;
-import org.apache.submarine.server.api.experiment.Submitter;
+import org.apache.submarine.server.api.Submitter;
import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.ExperimentLog;
+import org.apache.submarine.server.api.notebook.Notebook;
import org.apache.submarine.server.api.spec.ExperimentMeta;
import org.apache.submarine.server.api.spec.ExperimentSpec;
+import org.apache.submarine.server.api.spec.NotebookSpec;
+import org.apache.submarine.server.submitter.k8s.model.NotebookCR;
+import org.apache.submarine.server.submitter.k8s.parser.NotebookSpecParser;
import org.apache.submarine.server.submitter.k8s.util.MLJobConverter;
import org.apache.submarine.server.submitter.k8s.model.MLJob;
import org.apache.submarine.server.submitter.k8s.parser.ExperimentSpecParser;
+import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -227,6 +232,61 @@ public class K8sSubmitter implements Submitter {
return experimentLog;
}
+ @Override
+ public Notebook createNotebook(NotebookSpec spec) throws
SubmarineRuntimeException {
+ Notebook notebook;
+ try {
+ NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
+ Object object = api.createNamespacedCustomObject(notebookCR.getGroup(),
notebookCR.getVersion(),
+ notebookCR.getMetadata().getNamespace(), notebookCR.getPlural(),
notebookCR, "true");
+ notebook = parseResponseObject(object);
+ } catch (JsonSyntaxException e) {
+ LOG.error("K8s submitter: parse response object failed by " +
e.getMessage(), e);
+ throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream
response failed.");
+ } catch (ApiException e) {
+ LOG.error("K8s submitter: parse Notebook object failed by " +
e.getMessage(), e);
+ throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
+ }
+ return notebook;
+ }
+
+ @Override
+ public Notebook findNotebook(NotebookSpec spec) throws
SubmarineRuntimeException {
+ // TODO(ryan): Implement this method
+ return null;
+ }
+
+ @Override
+ public Notebook deleteNotebook(NotebookSpec spec) throws
SubmarineRuntimeException {
+ // TODO(ryan): Implement this method
+ return null;
+ }
+
+ private Notebook parseResponseObject(Object obj) throws
SubmarineRuntimeException {
+ Gson gson = new JSON().getGson();
+ String jsonString = gson.toJson(obj);
+ LOG.info("Upstream response JSON: {}", jsonString);
+ Notebook notebook;
+ try {
+ notebook = new Notebook();
+ NotebookCR notebookCR = gson.fromJson(jsonString, NotebookCR.class);
+ notebook.setUid(notebookCR.getMetadata().getUid());
+ notebook.setName(notebookCR.getMetadata().getName());
+ // notebook url
+ notebook.setUrl("/notebook/" + notebookCR.getMetadata().getNamespace() +
"/" +
+ notebookCR.getMetadata().getName());
+ DateTime createdTime = notebookCR.getMetadata().getCreationTimestamp();
+ if (createdTime != null) {
+ notebook.setCreatedTime(createdTime.toString());
+ notebook.setStatus(Notebook.Status.STATUS_CREATED.getValue());
+ }
+ } catch (JsonSyntaxException e) {
+ LOG.error("K8s submitter: parse response object failed by " +
e.getMessage(), e);
+ throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream
response failed.");
+ }
+ return notebook;
+ }
+
private String getJobLabelSelector(ExperimentSpec experimentSpec) {
// TODO(JohnTing): SELECTOR_KEY should be obtained from individual models
in MLJOB
if (experimentSpec.getMeta().getFramework()
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
new file mode 100644
index 0000000..96295d0
--- /dev/null
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.submitter.k8s.model;
+
+import com.google.gson.annotations.SerializedName;
+import io.kubernetes.client.models.V1ObjectMeta;
+
+public class NotebookCR {
+
+ public static final String CRD_NOTEBOOK_VERSION_V1 = "v1";
+ public static final String CRD_NOTEBOOK_GROUP_V1 = "kubeflow.org";
+ public static final String CRD_APIVERSION_V1 = CRD_NOTEBOOK_GROUP_V1 + "/" +
CRD_NOTEBOOK_VERSION_V1;
+ public static final String CRD_NOTEBOOK_KIND_V1 = "Notebook";
+ public static final String CRD_NOTEBOOK_PLURAL_V1 = "notebooks";
+
+ @SerializedName("apiVersion")
+ private String apiVersion;
+
+ @SerializedName("kind")
+ private String kind;
+
+ @SerializedName("metadata")
+ private V1ObjectMeta metadata;
+
+ private transient String group;
+
+ private transient String version;
+
+ private transient String plural;
+
+ @SerializedName("spec")
+ private NotebookCRSpec spec;
+
+ public NotebookCR() {
+ setApiVersion(CRD_APIVERSION_V1);
+ setKind(CRD_NOTEBOOK_KIND_V1);
+ setPlural(CRD_NOTEBOOK_PLURAL_V1);
+ setGroup(CRD_NOTEBOOK_GROUP_V1);
+ setVersion(CRD_NOTEBOOK_VERSION_V1);
+ }
+
+ public String getApiVersion() {
+ return apiVersion;
+ }
+
+ public void setApiVersion(String apiVersion) {
+ this.apiVersion = apiVersion;
+ }
+
+ public String getKind() {
+ return kind;
+ }
+
+ public void setKind(String kind) {
+ this.kind = kind;
+ }
+
+ public V1ObjectMeta getMetadata() {
+ return metadata;
+ }
+
+ public void setMetadata(V1ObjectMeta metadata) {
+ this.metadata = metadata;
+ }
+
+ public String getGroup() {
+ return this.group;
+ }
+
+ public void setGroup(String group) {
+ this.group = group;
+ }
+
+ public String getVersion() {
+ return this.version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public String getPlural() {
+ return this.plural;
+ }
+
+ public void setPlural(String plural) {
+ this.plural = plural;
+ }
+
+ public NotebookCRSpec getSpec() {
+ return spec;
+ }
+
+ public void setSpec(NotebookCRSpec spec) {
+ this.spec = spec;
+ }
+
+}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCRSpec.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCRSpec.java
new file mode 100644
index 0000000..05eb1dd
--- /dev/null
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCRSpec.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.submitter.k8s.model;
+
+import com.google.gson.annotations.SerializedName;
+import io.kubernetes.client.models.V1PodTemplateSpec;
+
+import java.math.BigDecimal;
+
+public class NotebookCRSpec {
+
+ public NotebookCRSpec() {
+
+ }
+
+ @SerializedName("template")
+ private V1PodTemplateSpec template;
+
+ /**
+ * Get the pod template
+ * @return pod template spec
+ */
+ public V1PodTemplateSpec getTemplate() {
+ return template;
+ }
+
+ /**
+ * Set the pod template
+ * @param template pod template
+ */
+ public void setTemplate(V1PodTemplateSpec template) {
+ this.template = template;
+ }
+
+ /**
+ * Get memory resource for container
+ * @return memory in Gi
+ */
+ public String getContainerMemory() {
+ V1PodTemplateSpec podSpec = getTemplate();
+ return String.join(" ",
+ podSpec.getSpec().getContainers().get(0)
+ .getResources().getLimits().get("memory").
+ getNumber().divide(BigDecimal.valueOf(1024 * 1024 *
1024)).toString() + "Gi");
+ }
+
+ /**
+ * Get CPU resource for container
+ * @return CPU in VCores
+ */
+ public String getContainerCpu() {
+ V1PodTemplateSpec podSpec = getTemplate();
+ return podSpec.getSpec().getContainers().get(0)
+ .getResources().getLimits().get("cpu").getNumber().toString();
+ }
+
+ /**
+ * Get GPU resource for container
+ * @return GPU
+ */
+ public String getContainerGpu() {
+ V1PodTemplateSpec podSpec = getTemplate();
+ return podSpec.getSpec().getContainers().get(0)
+
.getResources().getLimits().get("nvidia.com/gpu").getNumber().toString();
+ }
+
+ /**
+ * Get the image name
+ * @return image name
+ */
+ public String getContainerImageName() {
+ V1PodTemplateSpec podSpec = getTemplate();
+ return podSpec.getSpec().getContainers().get(0).getImage();
+ }
+}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
new file mode 100644
index 0000000..e606688
--- /dev/null
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.submitter.k8s.parser;
+
+import io.kubernetes.client.custom.Quantity;
+import io.kubernetes.client.models.V1Container;
+import io.kubernetes.client.models.V1EnvVar;
+import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.models.V1PodTemplateSpec;
+import io.kubernetes.client.models.V1PodSpec;
+import io.kubernetes.client.models.V1ResourceRequirements;
+
+import org.apache.submarine.commons.utils.SubmarineConfVars;
+import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.server.api.environment.Environment;
+import org.apache.submarine.server.api.spec.KernelSpec;
+import org.apache.submarine.server.api.spec.NotebookPodSpec;
+import org.apache.submarine.server.api.spec.NotebookSpec;
+import org.apache.submarine.server.environment.EnvironmentManager;
+import org.apache.submarine.server.submitter.k8s.model.NotebookCR;
+import org.apache.submarine.server.submitter.k8s.model.NotebookCRSpec;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class NotebookSpecParser {
+
+ private static SubmarineConfiguration conf =
+ SubmarineConfiguration.getInstance();
+
+
+ public static NotebookCR parseNotebook(NotebookSpec spec) {
+ NotebookCR notebookCR = new NotebookCR();
+ notebookCR.setMetadata(parseMetadata(spec));
+ notebookCR.setSpec(parseNotebookCRSpec(spec));
+ return notebookCR;
+ }
+
+ private static V1ObjectMeta parseMetadata(NotebookSpec spec) {
+ V1ObjectMeta meta = new V1ObjectMeta();
+ meta.setName(spec.getMeta().getName());
+ meta.setNamespace(spec.getMeta().getNamespace());
+ return meta;
+ }
+
+ private static NotebookCRSpec parseNotebookCRSpec(NotebookSpec spec) {
+ NotebookCRSpec CRSpec = new NotebookCRSpec();
+ CRSpec.setTemplate(parseTemplateSpec(spec));
+ return CRSpec;
+ }
+
+ private static V1PodTemplateSpec parseTemplateSpec(NotebookSpec
notebookSpec) {
+ NotebookPodSpec notebookPodSpec = notebookSpec.getSpec();
+ V1PodTemplateSpec podTemplateSpec = new V1PodTemplateSpec();
+ V1PodSpec podSpec = new V1PodSpec();
+ // Set container
+ List<V1Container> containers = new ArrayList<>();
+ V1Container container = new V1Container();
+ container.setName(notebookSpec.getMeta().getName());
+
+ // Environment variables
+ if (notebookPodSpec.getEnvVars() == null) {
+ container.setEnv(parseEnvVars(notebookPodSpec));
+ }
+
+ // Environment
+ if (getEnvironment(notebookSpec) != null) {
+ String baseImage =
getEnvironment(notebookSpec).getEnvironmentSpec().getDockerImage();
+ KernelSpec kernel =
getEnvironment(notebookSpec).getEnvironmentSpec().getKernelSpec();
+ container.setImage(baseImage);
+ if (kernel.getDependencies().size() > 0) {
+ String condaVersionValidationCommand =
generateCondaVersionValidateCommand();
+ StringBuffer createCommand = new StringBuffer();
+ String condaEnvironmentName = kernel.getName();
+
+ createCommand.append("conda create -q -y -n " + condaEnvironmentName);
+ for (String channel : kernel.getChannels()) {
+ createCommand.append(" ");
+ createCommand.append("-c");
+ createCommand.append(" ");
+ createCommand.append(channel);
+ }
+ for (String dependency : kernel.getDependencies()) {
+ createCommand.append(" ");
+ createCommand.append(dependency);
+ }
+
+ String activateEnvCommand = "source activate " + condaEnvironmentName;
+ String pathCommand = "PATH=/opt/conda/envs/env/bin:$PATH";
+ String finalCommand = condaVersionValidationCommand +
+ " && " + createCommand.toString() + " && "
+ + activateEnvCommand + " && " + pathCommand;
+ V1EnvVar envCommand = new V1EnvVar();
+ envCommand.setName("ENVIRONMENT_COMMAND");
+ envCommand.setValue(finalCommand);
+ container.addEnvItem(envCommand);
+ }
+ }
+
+ // Resources
+ if (notebookPodSpec.getResources() != null) {
+ V1ResourceRequirements resources = new V1ResourceRequirements();
+ resources.setLimits(parseResources(notebookPodSpec));
+ container.setResources(resources);
+ }
+
+ containers.add(container);
+ podSpec.setContainers(containers);
+
+ podTemplateSpec.setSpec(podSpec);
+ return podTemplateSpec;
+ }
+
+ private static List<V1EnvVar> parseEnvVars(NotebookPodSpec podSpec) {
+ if (podSpec.getEnvVars() == null)
+ return null;
+ List<V1EnvVar> envVars = new ArrayList<>();
+ for (Map.Entry<String, String> entry : podSpec.getEnvVars().entrySet()) {
+ V1EnvVar env = new V1EnvVar();
+ env.setName(entry.getKey());
+ env.setValue(entry.getValue());
+ envVars.add(env);
+ }
+ return envVars;
+ }
+
+ private static Map<String, Quantity> parseResources(NotebookPodSpec podSpec)
{
+
+ Map<String, Quantity> resources = new HashMap<>();
+ podSpec.setResources(podSpec.getResources());
+
+ if (podSpec.getCpu() != null) {
+ resources.put("cpu", new Quantity(podSpec.getCpu()));
+ }
+ if (podSpec.getMemory() != null) {
+ resources.put("memory", new Quantity(podSpec.getMemory()));
+ }
+ if (podSpec.getGpu() != null) {
+ resources.put("nvidia.com/gpu", new Quantity(podSpec.getGpu()));
+ }
+ return resources;
+ }
+
+ private static Environment getEnvironment(NotebookSpec notebookSpec) {
+ if (notebookSpec.getEnvironment().getName() != null) {
+ EnvironmentManager environmentManager = EnvironmentManager.getInstance();
+ return environmentManager
+ .getEnvironment(notebookSpec.getEnvironment().getName());
+ } else {
+ return null;
+ }
+ }
+
+ private static String generateCondaVersionValidateCommand() {
+ String currentVersion = "currentVersion=$(conda -V | cut -f2 -d' ');";
+ String minVersion = "minVersion=\""
+ +
conf.getString(SubmarineConfVars.ConfVars.ENVIRONMENT_CONDA_MIN_VERSION)
+ + "\";";
+ String maxVersion = "maxVersion=\""
+ +
conf.getString(SubmarineConfVars.ConfVars.ENVIRONMENT_CONDA_MAX_VERSION)
+ + "\";";
+ StringBuffer condaVersionValidationCommand = new StringBuffer();
+ condaVersionValidationCommand.append(minVersion);
+ condaVersionValidationCommand.append(maxVersion);
+ condaVersionValidationCommand.append(currentVersion);
+ condaVersionValidationCommand.append("if [ \"$(printf '%s\\n' "
+ + "\"$minVersion\" \"$maxVersion\" \"$currentVersion\" | sort -V "
+ + "| head -n2 | tail -1 )\" != \"$currentVersion\" ]; then echo "
+ + "\"Conda version should be between " + minVersion + " and "
+ + maxVersion + "\"; exit 1; else echo \"Conda current version is "
+ + currentVersion + ". Moving forward with env creation and "
+ + "activation.\"; fi");
+ return condaVersionValidationCommand.toString();
+ }
+}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
index 8517671..c44bec9 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
@@ -49,14 +49,14 @@ import io.kubernetes.client.models.V1Container;
public class ExperimentSpecParserTest extends SpecBuilder {
-
+
private static SubmarineConfiguration conf =
SubmarineConfiguration.getInstance();
-
+
@Test
public void testValidTensorFlowExperiment() throws IOException,
URISyntaxException, InvalidSpecException {
- ExperimentSpec experimentSpec = buildFromJsonFile(tfJobReqFile);
+ ExperimentSpec experimentSpec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, tfJobReqFile);
TFJob tfJob = (TFJob) ExperimentSpecParser.parseJob(experimentSpec);
validateMetadata(experimentSpec.getMeta(), tfJob.getMetadata(),
ExperimentMeta.SupportedMLFramework.TENSORFLOW.getName().toLowerCase()
@@ -74,7 +74,7 @@ public class ExperimentSpecParserTest extends SpecBuilder {
@Test
public void testInvalidTensorFlowExperiment() throws IOException,
URISyntaxException {
- ExperimentSpec experimentSpec = buildFromJsonFile(tfJobReqFile);
+ ExperimentSpec experimentSpec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, tfJobReqFile);
// Case 1. Invalid framework name
experimentSpec.getMeta().setFramework("fooframework");
try {
@@ -85,7 +85,7 @@ public class ExperimentSpecParserTest extends SpecBuilder {
}
// Case 2. Invalid TensorFlow replica name. It can only be "ps" "worker"
"chief" and "Evaluator"
- experimentSpec = buildFromJsonFile(tfJobReqFile);
+ experimentSpec = (ExperimentSpec) buildFromJsonFile(ExperimentSpec.class,
tfJobReqFile);
experimentSpec.getSpec().put("foo",
experimentSpec.getSpec().get(TFJobReplicaType.Ps.getTypeName()));
experimentSpec.getSpec().remove(TFJobReplicaType.Ps.getTypeName());
try {
@@ -99,7 +99,8 @@ public class ExperimentSpecParserTest extends SpecBuilder {
@Test
public void testValidPyTorchExperiment() throws IOException,
URISyntaxException, InvalidSpecException {
- ExperimentSpec experimentSpec = buildFromJsonFile(pytorchJobReqFile);
+ ExperimentSpec experimentSpec =
+ (ExperimentSpec) buildFromJsonFile(ExperimentSpec.class,
pytorchJobReqFile);
PyTorchJob pyTorchJob = (PyTorchJob)
ExperimentSpecParser.parseJob(experimentSpec);
validateMetadata(experimentSpec.getMeta(), pyTorchJob.getMetadata(),
ExperimentMeta.SupportedMLFramework.PYTORCH.getName().toLowerCase()
@@ -117,7 +118,8 @@ public class ExperimentSpecParserTest extends SpecBuilder {
@Test
public void testInvalidPyTorchJobSpec() throws IOException,
URISyntaxException {
- ExperimentSpec experimentSpec = buildFromJsonFile(pytorchJobReqFile);
+ ExperimentSpec experimentSpec =
+ (ExperimentSpec) buildFromJsonFile(ExperimentSpec.class,
pytorchJobReqFile);
// Case 1. Invalid framework name
experimentSpec.getMeta().setFramework("fooframework");
try {
@@ -128,7 +130,7 @@ public class ExperimentSpecParserTest extends SpecBuilder {
}
// Case 2. Invalid PyTorch replica name. It can only be "master" and
"worker"
- experimentSpec = buildFromJsonFile(pytorchJobReqFile);
+ experimentSpec = (ExperimentSpec) buildFromJsonFile(ExperimentSpec.class,
pytorchJobReqFile);
experimentSpec.getSpec().put("ps", experimentSpec.getSpec().get(
PyTorchJobReplicaType.Master.getTypeName()));
experimentSpec.getSpec().remove(PyTorchJobReplicaType.Master.getTypeName());
@@ -188,7 +190,7 @@ public class ExperimentSpecParserTest extends SpecBuilder {
Assert.assertEquals(expectedMasterContainerCpu,
actualMasterContainerCpu);
}
-
+
@Test
public void testValidPyTorchJobSpecWithEnv()
throws IOException, URISyntaxException, InvalidSpecException {
@@ -217,7 +219,8 @@ public class ExperimentSpecParserTest extends SpecBuilder {
environmentManager.createEnvironment(spec);
- ExperimentSpec jobSpec = buildFromJsonFile(pytorchJobWithEnvReqFile);
+ ExperimentSpec jobSpec =
+ (ExperimentSpec) buildFromJsonFile(ExperimentSpec.class,
pytorchJobWithEnvReqFile);
PyTorchJob pyTorchJob = (PyTorchJob)
ExperimentSpecParser.parseJob(jobSpec);
MLJobReplicaSpec mlJobReplicaSpec = pyTorchJob.getSpec().getReplicaSpecs()
@@ -230,7 +233,7 @@ public class ExperimentSpecParserTest extends SpecBuilder {
Assert.assertEquals("/bin/bash", initContainer.getCommand().get(0));
Assert.assertEquals("-c", initContainer.getCommand().get(1));
-
+
String minVersion = "minVersion=\""
+ conf.getString(
SubmarineConfVars.ConfVars.ENVIRONMENT_CONDA_MIN_VERSION)
@@ -241,20 +244,20 @@ public class ExperimentSpecParserTest extends SpecBuilder
{
+ "\";";
String currentVersion = "currentVersion=$(conda -V | cut -f2 -d' ');";
Assert.assertEquals(
- minVersion + maxVersion + currentVersion
+ minVersion + maxVersion + currentVersion
+ "if [ \"$(printf '%s\\n' \"$minVersion\" \"$maxVersion\" "
+ "\"$currentVersion\" | sort -V | head -n2 | tail -1 )\" "
- + "!= \"$currentVersion\" ]; then echo \"Conda version " +
- "should be between minVersion=\"4.0.1\"; " +
+ + "!= \"$currentVersion\" ]; then echo \"Conda version " +
+ "should be between minVersion=\"4.0.1\"; " +
"and maxVersion=\"4.10.10\";\"; exit 1; else echo "
+ "\"Conda current version is " + currentVersion + ". "
+ "Moving forward with env creation and activation.\";
"
- + "fi && " +
+ + "fi && " +
"conda create -n " + kernelName + " -c " + channel + " " + dependency
+ " && " + "echo \"source activate " + kernelName + "\" >
~/.bashrc"
+ " && " + "PATH=/opt/conda/envs/env/bin:$PATH",
initContainer.getCommand().get(2));
-
+
environmentManager.deleteEnvironment(envName);
}
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
index 0a232ce..a27f88e 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
@@ -59,14 +59,14 @@ public class K8SJobSubmitterTest extends SpecBuilder {
@Test
public void testRunPyTorchJobPerRequest() throws URISyntaxException,
IOException, SubmarineRuntimeException {
- ExperimentSpec spec = buildFromJsonFile(pytorchJobReqFile);
+ ExperimentSpec spec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, pytorchJobReqFile);
run(spec);
}
@Test
public void testRunTFJobPerRequest() throws URISyntaxException,
IOException, SubmarineRuntimeException {
- ExperimentSpec spec = buildFromJsonFile(tfJobReqFile);
+ ExperimentSpec spec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, tfJobReqFile);
run(spec);
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/MLJobConverterTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/MLJobConverterTest.java
index 75267ba..851f658 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/MLJobConverterTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/MLJobConverterTest.java
@@ -45,7 +45,7 @@ public class MLJobConverterTest extends SpecBuilder {
@Test
public void testMLJob2Job() throws IOException, URISyntaxException,
InvalidSpecException {
// Accepted Status
- ExperimentSpec spec = buildFromJsonFile(tfJobReqFile);
+ ExperimentSpec spec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, tfJobReqFile);
MLJob mlJob = ExperimentSpecParser.parseJob(spec);
V1JobStatus status = new V1JobStatusBuilder().build();
mlJob.setStatus(status);
@@ -98,7 +98,7 @@ public class MLJobConverterTest extends SpecBuilder {
@Test
public void testMLJob2DeleteOptions() throws IOException, URISyntaxException,
InvalidSpecException {
- ExperimentSpec spec = buildFromJsonFile(tfJobReqFile);
+ ExperimentSpec spec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, tfJobReqFile);
MLJob mlJob = ExperimentSpecParser.parseJob(spec);
V1DeleteOptions options = MLJobConverter.toDeleteOptionsFromMLJob(mlJob);
Assert.assertNotNull(options);
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
new file mode 100644
index 0000000..05de939
--- /dev/null
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.submitter.k8s;
+
+import io.kubernetes.client.models.V1ObjectMeta;
+import org.apache.submarine.server.api.spec.NotebookMeta;
+import org.apache.submarine.server.api.spec.NotebookPodSpec;
+import org.apache.submarine.server.api.spec.NotebookSpec;
+import org.apache.submarine.server.submitter.k8s.model.NotebookCR;
+import org.apache.submarine.server.submitter.k8s.model.NotebookCRSpec;
+import org.apache.submarine.server.submitter.k8s.parser.NotebookSpecParser;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+public class NotebookSpecParserTest extends SpecBuilder {
+
+ @Test
+ public void testValidNotebook() throws IOException, URISyntaxException {
+ NotebookSpec notebookSpec = (NotebookSpec)
buildFromJsonFile(NotebookSpec.class, notebookReqFile);
+ NotebookCR notebook = NotebookSpecParser.parseNotebook(notebookSpec);
+
+ validateMetadata(notebookSpec.getMeta(), notebook.getMetadata());
+ validateEnvironment(notebookSpec, notebook.getSpec());
+ validatePodSpec(notebookSpec.getSpec(), notebook);
+ }
+
+ private void validateMetadata(NotebookMeta meta, V1ObjectMeta actualMeta) {
+ Assert.assertEquals(meta.getName(), actualMeta.getName());
+ Assert.assertEquals(meta.getNamespace(), actualMeta.getNamespace());
+ }
+
+ private void validateEnvironment(NotebookSpec spec, NotebookCRSpec
actualPodSpec) {
+ String expectedImage = spec.getEnvironment().getImage();
+ String actualImage = actualPodSpec.getContainerImageName();
+ Assert.assertEquals(expectedImage, actualImage);
+ }
+
+ private void validatePodSpec(NotebookPodSpec podSpec, NotebookCR notebook) {
+ NotebookCRSpec notebookCRSpec = null;
+ if (notebook != null) {
+ notebookCRSpec = notebook.getSpec();
+ }
+ Assert.assertNotNull(notebookCRSpec);
+
+ // mem
+ String expectedContainerMem = podSpec.getMemory();
+ String actualContainerMem = notebookCRSpec.getContainerMemory();
+ Assert.assertEquals(expectedContainerMem, actualContainerMem);
+
+ // cpu
+ String expectedContainerCpu = podSpec.getCpu();
+ String actualContainerCpu = notebookCRSpec.getContainerCpu();
+ Assert.assertEquals(expectedContainerCpu, actualContainerCpu);
+ }
+
+}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
index 50c83f8..11546ab 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
@@ -30,6 +30,7 @@ import java.nio.file.Files;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.submarine.server.api.spec.ExperimentSpec;
+import org.apache.submarine.server.api.spec.NotebookSpec;
public abstract class SpecBuilder {
// The spec files in test/resources
@@ -38,13 +39,18 @@ public abstract class SpecBuilder {
protected final String pytorchJobWithEnvReqFile =
"/pytorch_job_req_env.json";
protected final String pytorchJobWithInvalidEnvReqFile =
"/pytorch_job_req_invalid_env.json";
+ protected final String notebookReqFile = "/notebook_req.json";
- protected ExperimentSpec buildFromJsonFile(String filePath) throws
IOException,
+ protected Object buildFromJsonFile(Object obj, String filePath) throws
IOException,
URISyntaxException {
Gson gson = new GsonBuilder().create();
try (Reader reader =
Files.newBufferedReader(getCustomJobSpecFile(filePath).toPath(),
StandardCharsets.UTF_8)) {
- return gson.fromJson(reader, ExperimentSpec.class);
+ if (obj.equals(NotebookSpec.class)) {
+ return gson.fromJson(reader, NotebookSpec.class);
+ } else {
+ return gson.fromJson(reader, ExperimentSpec.class);
+ }
}
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/resources/notebook_req.json
b/submarine-server/server-submitter/submitter-k8s/src/test/resources/notebook_req.json
new file mode 100644
index 0000000..e1dace6
--- /dev/null
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/resources/notebook_req.json
@@ -0,0 +1,15 @@
+{
+ "meta": {
+ "name": "my-nb",
+ "namespace": "default"
+ },
+ "environment": {
+ "image": "apache/submarine:tf2.1.0-jupyter"
+ },
+ "spec": {
+ "envVars": {
+ "env": "env"
+ },
+ "resources": "cpu=1,memory=1Gi"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@submarine.apache.org
For additional commands, e-mail: dev-h...@submarine.apache.org
![https://user-images.githubusercontent.com/52355146/88690339-248ad300-d12e-11ea-9cb8-9c9e7f4a5b1c.png"](https://user-images.githubusercontent.com/52355146/88690339-248ad300-d12e-11ea-9cb8-9c9e7f4a5b1c.png"){kind=link}
![https://user-images.githubusercontent.com/52355146/88690409-39676680-d12e-11ea-8102-b3d92dd99c80.png"](https://user-images.githubusercontent.com/52355146/88690409-39676680-d12e-11ea-8102-b3d92dd99c80.png"){kind=link}