This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new e7d22d1 SUBMARINE-1127. Jupyter Notebook can support initing
overwrite.json
e7d22d1 is described below
commit e7d22d1488799b45cc6276931938859db52cd8a1
Author: Thinking <[email protected]>
AuthorDate: Sun Jan 9 13:00:26 2022 +0800
SUBMARINE-1127. Jupyter Notebook can support initing overwrite.json
### What is this PR for?
Jupyter Notebook have created a workspace volume /home/jovyan/workspace,
but in some cases, we may need to do some initialization configuration when
starting notebook. For example, we can configure an overrides.json in
/opt/conda/share/jupyter/lab/settings/.
We can refer to the official document of jupyter about overrides.json.
<https://jupyterlab.readthedocs.io/en/stable/user/directories.html#overrides-json>
Assuming that the user of submarine needs to specify the language of the
initialized notebook to Chinese, we can write this in overrides.json.
```json
{
"jupyterlab/translation-extension:plugin": {
"locale": "zh_CN"
}
}
```
After this configuration, the new logging will be displayed in Chinese by
default.
### What type of PR is it?
Improvement
### Todos
* [x] - Add a configmap to use `overwrite.json` as a volume config file in
pod
### What is the Jira issue?
https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-1127?filter=allissues
### How should this be tested?
Try to set an env named `SUBMARINE_NOTEBOOK_DEFAULT_OVERWRITE_JSON` in
submarine-server deployment.
### Screenshots (if appropriate)
No
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? Yes
Author: Thinking <[email protected]>
Signed-off-by: Kevin <[email protected]>
Closes #844 from cdmikechen/SUBMARINE-1127 and squashes the following
commits:
584ea538 [Thinking] SUBMARINE-1127. Jupyter Notebook can support initing
overwrite.json
---
.../artifacts/submarine/submarine-rbac.yaml | 3 +-
.../submarine/commons/utils/SubmarineConfVars.java | 2 +
.../server/submitter/k8s/K8sSubmitter.java | 81 ++++++++++++++++++++++
.../submitter/k8s/parser/ConfigmapSpecParser.java | 65 +++++++++++++++++
.../submitter/k8s/parser/NotebookSpecParser.java | 34 +++++++--
.../server/submitter/k8s/util/NotebookUtils.java | 2 +
.../submitter/k8s/NotebookSpecParserTest.java | 17 +++++
7 files changed, 199 insertions(+), 5 deletions(-)
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-rbac.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-rbac.yaml
index 58d8264..08eaac1 100644
--- a/submarine-cloud-v2/artifacts/submarine/submarine-rbac.yaml
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-rbac.yaml
@@ -79,7 +79,7 @@ rules:
- delete
- deletecollection
- patch
- - update
+ - update
- apiGroups:
- ""
resources:
@@ -88,6 +88,7 @@ rules:
- services
- persistentvolumeclaims
- events
+ - configmaps
verbs:
- '*'
- apiGroups:
diff --git
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
index ed9a35f..42622e0 100644
---
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
+++
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
@@ -63,6 +63,8 @@ public class SubmarineConfVars {
METASTORE_JDBC_USERNAME("metastore.jdbc.username", "metastore"),
METASTORE_JDBC_PASSWORD("metastore.jdbc.password", "password"),
+
SUBMARINE_NOTEBOOK_DEFAULT_OVERWRITE_JSON("submarine.notebook.default.overwrite_json",
""),
+
WORKBENCH_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE(
"workbench.websocket.max.text.message.size", "1024000"),
WORKBENCH_WEB_WAR("workbench.web.war",
"submarine-workbench/workbench-web/dist"),
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
index cae04f5..b5e44c8 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
@@ -41,6 +41,7 @@ import io.kubernetes.client.openapi.JSON;
import io.kubernetes.client.openapi.apis.AppsV1Api;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.apis.CustomObjectsApi;
+import io.kubernetes.client.openapi.models.V1ConfigMap;
import io.kubernetes.client.openapi.models.V1DeleteOptionsBuilder;
import io.kubernetes.client.openapi.models.V1Deployment;
import io.kubernetes.client.openapi.models.CoreV1Event;
@@ -54,6 +55,8 @@ import io.kubernetes.client.util.Watch;
import io.kubernetes.client.util.ClientBuilder;
import io.kubernetes.client.util.KubeConfig;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.serve.istio.IstioVirtualService;
@@ -78,6 +81,7 @@ import
org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRoute
import org.apache.submarine.server.submitter.k8s.model.ingressroute.SpecRoute;
import org.apache.submarine.server.submitter.k8s.model.pytorchjob.PyTorchJob;
import org.apache.submarine.server.submitter.k8s.model.tfjob.TFJob;
+import org.apache.submarine.server.submitter.k8s.parser.ConfigmapSpecParser;
import org.apache.submarine.server.submitter.k8s.parser.ExperimentSpecParser;
import org.apache.submarine.server.submitter.k8s.parser.NotebookSpecParser;
import org.apache.submarine.server.submitter.k8s.parser.VolumeSpecParser;
@@ -101,6 +105,14 @@ public class K8sSubmitter implements Submitter {
private static final String ENV_NAMESPACE = "ENV_NAMESPACE";
+ private static final String OVERWRITE_JSON;
+
+ static {
+ final SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
+ OVERWRITE_JSON = conf.getString(
+
SubmarineConfVars.ConfVars.SUBMARINE_NOTEBOOK_DEFAULT_OVERWRITE_JSON);
+ }
+
// K8s API client for CRD
private CustomObjectsApi api;
@@ -385,6 +397,7 @@ public class K8sSubmitter implements Submitter {
final String host = NotebookUtils.HOST_PATH;
final String workspacePvc = String.format("%s-%s",
NotebookUtils.PVC_PREFIX, name);
final String userPvc = String.format("%s-user-%s",
NotebookUtils.PVC_PREFIX, name);
+ final String configmap = String.format("%s-%s",
NotebookUtils.OVERWRITE_PREFIX, name);
String namespace = getServerNamespace();
// parse notebook custom resource
@@ -412,6 +425,23 @@ public class K8sSubmitter implements Submitter {
"Notebook object failed by " + e.getMessage());
}
+ // create configmap if needed
+ boolean needOverwrite = StringUtils.isNotBlank(OVERWRITE_JSON);
+ if (needOverwrite) {
+ try {
+ createConfigMap(configmap, namespace,
NotebookUtils.DEFAULT_OVERWRITE_FILE_NAME, OVERWRITE_JSON);
+ } catch (JsonSyntaxException e) {
+ LOG.error("K8s submitter: parse response object failed by " +
e.getMessage(), e);
+ rollbackCreationPVC(namespace, workspacePvc, userPvc);
+ throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream
response failed.");
+ } catch (ApiException e) {
+ LOG.error("K8s submitter: parse Notebook object failed by " +
e.getMessage(), e);
+ rollbackCreationPVC(namespace, workspacePvc, userPvc);
+ throw new SubmarineRuntimeException(e.getCode(), "K8s submitter: parse
Notebook object failed by " +
+ e.getMessage());
+ }
+ }
+
// create notebook custom resource
try {
Object object = api.createNamespacedCustomObject(notebookCR.getGroup(),
notebookCR.getVersion(),
@@ -419,10 +449,12 @@ public class K8sSubmitter implements Submitter {
notebook = NotebookUtils.parseObject(object,
NotebookUtils.ParseOpt.PARSE_OPT_CREATE);
} catch (JsonSyntaxException e) {
LOG.error("K8s submitter: parse response object failed by " +
e.getMessage(), e);
+ if (needOverwrite) rollbackCreationConfigMap(namespace, configmap);
rollbackCreationPVC(namespace, workspacePvc, userPvc);
throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream
response failed.");
} catch (ApiException e) {
LOG.error("K8s submitter: parse Notebook object failed by " +
e.getMessage(), e);
+ if (needOverwrite) rollbackCreationConfigMap(namespace, configmap);
rollbackCreationPVC(namespace, workspacePvc, userPvc);
throw new SubmarineRuntimeException(e.getCode(), "K8s submitter: parse
Notebook object failed by " +
e.getMessage());
@@ -435,6 +467,7 @@ public class K8sSubmitter implements Submitter {
LOG.error("K8s submitter: Create ingressroute for Notebook object failed
by " +
e.getMessage(), e);
rollbackCreationNotebook(notebookCR, namespace);
+ if (needOverwrite) rollbackCreationConfigMap(namespace, configmap);
rollbackCreationPVC(namespace, workspacePvc, userPvc);
throw new SubmarineRuntimeException(e.getCode(), "K8s submitter:
ingressroute for Notebook " +
"object failed by " + e.getMessage());
@@ -500,6 +533,11 @@ public class K8sSubmitter implements Submitter {
deletePersistentVolumeClaim(String.format("%s-%s",
NotebookUtils.PVC_PREFIX, name), namespace);
// user set pvc
deletePersistentVolumeClaim(String.format("%s-user-%s",
NotebookUtils.PVC_PREFIX, name), namespace);
+
+ // configmap
+ if (StringUtils.isNoneBlank(OVERWRITE_JSON)) {
+ deleteConfigMap(namespace, String.format("%s-%s",
NotebookUtils.OVERWRITE_PREFIX, name));
+ }
} catch (ApiException e) {
throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
}
@@ -806,6 +844,49 @@ public class K8sSubmitter implements Submitter {
return seldonDeployment;
}
+ /**
+ * Create ConfigMap with values (key1, value1, key2, value2, ...)
+ */
+ public void createConfigMap(String name, String namespace, String ... values)
+ throws ApiException {
+ V1ConfigMap configMap = ConfigmapSpecParser.parseConfigMap(name, values);
+
configMap.getMetadata().setOwnerReferences(OwnerReferenceUtils.getOwnerReference());
+ try {
+ coreApi.createNamespacedConfigMap(namespace, configMap, "true", null,
null);
+ } catch (ApiException e) {
+ LOG.error("Exception when creating configmap " + e.getMessage(), e);
+ throw e;
+ }
+ }
+
+ /**
+ * Delete ConfigMap
+ */
+ public void deleteConfigMap(String namespace, String name) throws
ApiException {
+ try {
+ coreApi.deleteNamespacedConfigMap(name, namespace,
+ "true", null, null, null,
+ null, null);
+ } catch (ApiException e) {
+ LOG.error("Exception when deleting config map " + e.getMessage(), e);
+ throw e;
+ }
+ }
+
+ /**
+ * Rollback to delete ConfigMap
+ */
+ private void rollbackCreationConfigMap(String namespace, String ... names)
+ throws SubmarineRuntimeException {
+ try {
+ for (String name : names) {
+ deleteConfigMap(namespace, name);
+ }
+ } catch (ApiException e) {
+ throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
+ }
+ }
+
private void rollbackCreationPVC(String namespace, String ... pvcNames) {
try {
for (String pvcName : pvcNames) {
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ConfigmapSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ConfigmapSpecParser.java
new file mode 100644
index 0000000..46dcf62
--- /dev/null
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ConfigmapSpecParser.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.submitter.k8s.parser;
+
+
+import io.kubernetes.client.openapi.models.V1ConfigMap;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class ConfigmapSpecParser {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(ConfigmapSpecParser.class);
+
+ public static V1ConfigMap parseConfigMap(String name, String... values) {
+ Map<String, String> datas = new LinkedHashMap<>();
+ for (int i = 0, size = values.length; i < size; i += 2) {
+ try {
+ datas.put(values[i], values[i + 1]);
+ } catch (ArrayIndexOutOfBoundsException e) {// Avoid values by odd
numbers
+ LOG.warn("Can not find ConfigMap value in index[{}], skip this value",
i + 1);
+ }
+ }
+ return parseConfigMap(name, datas);
+ }
+
+ public static V1ConfigMap parseConfigMap(String name, Map<String, String>
datas) {
+ V1ConfigMap configMap = new V1ConfigMap();
+ /*
+ Required value
+ 1. metadata.name
+ 2. spec.data
+ 3. spec.resources
+ Others are not necessary
+ */
+
+ V1ObjectMeta metadata = new V1ObjectMeta();
+ metadata.setName(name);
+ configMap.setMetadata(metadata);
+ configMap.data(datas);
+
+ return configMap;
+ }
+
+}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
index 2067364..a0533f4 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
@@ -20,6 +20,7 @@
package org.apache.submarine.server.submitter.k8s.parser;
import io.kubernetes.client.custom.Quantity;
+import io.kubernetes.client.openapi.models.V1ConfigMapVolumeSource;
import io.kubernetes.client.openapi.models.V1Container;
import io.kubernetes.client.openapi.models.V1EnvVar;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
@@ -30,6 +31,7 @@ import io.kubernetes.client.openapi.models.V1Volume;
import io.kubernetes.client.openapi.models.V1VolumeMount;
import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimVolumeSource;
+import org.apache.commons.lang3.StringUtils;
import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.server.api.environment.Environment;
@@ -53,6 +55,9 @@ public class NotebookSpecParser {
private static final String DEFAULT_WORKSPACE_MOUNT_PATH =
"/home/jovyan/workspace";
// jupyter user setting path, avoid losing user setting after pod restarted
private static final String DEFAULT_USER_SET_MOUNT_PATH =
"/home/jovyan/.jupyter";
+ // overrides.json application settings directory
+ //
https://jupyterlab.readthedocs.io/en/stable/user/directories.html#overrides-json
+ private static final String DEFAULT_APPLICATION_SETTING_PATH =
"/opt/conda/share/jupyter/lab/settings";
private static final SubmarineConfiguration conf =
SubmarineConfiguration.getInstance();
@@ -165,10 +170,6 @@ public class NotebookSpecParser {
userSetting.setMountPath(DEFAULT_USER_SET_MOUNT_PATH);
userSetting.setName(String.format("%s-user-%s",
NotebookUtils.STORAGE_PREFIX, name));
volumeMountList.add(userSetting);
- container.setVolumeMounts(volumeMountList);
-
- containers.add(container);
- podSpec.setContainers(containers);
// create volume object for persistent volume
List<V1Volume> volumeList = new ArrayList<>();
@@ -187,6 +188,31 @@ public class NotebookSpecParser {
userVolume.setPersistentVolumeClaim(userPvc);
volumeList.add(userVolume);
+ // add overwrite.json configmap
+ String overwriteJson = conf.getString(
+
SubmarineConfVars.ConfVars.SUBMARINE_NOTEBOOK_DEFAULT_OVERWRITE_JSON);
+ if (StringUtils.isNotBlank(overwriteJson)) {
+ // Volume Mount
+ V1VolumeMount overwriteVm = new V1VolumeMount();
+ overwriteVm.setMountPath(String.format("%s/%s",
DEFAULT_APPLICATION_SETTING_PATH,
+ NotebookUtils.DEFAULT_OVERWRITE_FILE_NAME));
+ overwriteVm.setSubPath(NotebookUtils.DEFAULT_OVERWRITE_FILE_NAME);
+ overwriteVm.setName(String.format("%s-%s",
NotebookUtils.OVERWRITE_PREFIX, name));
+ volumeMountList.add(overwriteVm);
+
+ // Volume
+ V1Volume overwriteVolume = new V1Volume();
+ overwriteVolume.setName(String.format("%s-%s",
NotebookUtils.OVERWRITE_PREFIX, name));
+ V1ConfigMapVolumeSource overwriteCm = new V1ConfigMapVolumeSource();
+ overwriteCm.setName(String.format("%s-%s",
NotebookUtils.OVERWRITE_PREFIX, name));
+ overwriteVolume.setConfigMap(overwriteCm);
+ volumeList.add(overwriteVolume);
+ }
+
+ // add volume mounts and volumes
+ container.setVolumeMounts(volumeMountList);
+ containers.add(container);
+ podSpec.setContainers(containers);
podSpec.setVolumes(volumeList);
podTemplateSpec.setSpec(podSpec);
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
index 8805997..504c21d 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
@@ -49,7 +49,9 @@ public class NotebookUtils {
public static final String STORAGE_PREFIX = "notebook-storage";
public static final String PV_PREFIX = "notebook-pv";
public static final String PVC_PREFIX = "notebook-pvc";
+ public static final String OVERWRITE_PREFIX = "overwrite-configmap";
public static final String HOST_PATH = "/mnt";
+ public static final String DEFAULT_OVERWRITE_FILE_NAME = "overrides.json";
public enum ParseOpt {
PARSE_OPT_CREATE,
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
index 471301a..2bf0564 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
@@ -19,6 +19,7 @@
package org.apache.submarine.server.submitter.k8s;
+import io.kubernetes.client.openapi.models.V1ConfigMap;
import io.kubernetes.client.openapi.models.V1EnvVar;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import org.apache.submarine.server.api.spec.NotebookMeta;
@@ -26,6 +27,7 @@ import org.apache.submarine.server.api.spec.NotebookPodSpec;
import org.apache.submarine.server.api.spec.NotebookSpec;
import org.apache.submarine.server.submitter.k8s.model.NotebookCR;
import org.apache.submarine.server.submitter.k8s.model.NotebookCRSpec;
+import org.apache.submarine.server.submitter.k8s.parser.ConfigmapSpecParser;
import org.apache.submarine.server.submitter.k8s.parser.NotebookSpecParser;
import org.junit.Assert;
import org.junit.Test;
@@ -85,4 +87,19 @@ public class NotebookSpecParserTest extends SpecBuilder {
Assert.assertEquals(expectedContainerCpu, actualContainerCpu);
}
+ @Test
+ public void testConfigMap() {
+ String overwriteJson = "{ \"@jupyterlab/translation-extension:plugin\": " +
+ "{ \"locale\": \"zh_CN\" } }";
+ V1ConfigMap configMap = ConfigmapSpecParser.parseConfigMap("test",
+ "overwrite.json", overwriteJson);
+ Map<String, String> data = configMap.getData();
+ Assert.assertEquals(data.size(), 1);
+ Assert.assertEquals(data.get("overwrite.json"), overwriteJson);
+
+ V1ConfigMap configMap2 = ConfigmapSpecParser.parseConfigMap("test", data);
+ Map<String, String> data2 = configMap2.getData();
+ Assert.assertEquals(data2.size(), 1);
+ Assert.assertEquals(data2.get("overwrite.json"), overwriteJson);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
