This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new bdf6147 SUBMARINE-964. Pull notebook image is very time consuming
bdf6147 is described below
commit bdf614751c616cbf46f320db2fd6619ef64d12a8
Author: Brandon Lin <[email protected]>
AuthorDate: Sat Sep 4 23:15:29 2021 +0800
SUBMARINE-964. Pull notebook image is very time consuming
### What is this PR for?
Pulling the image of notebook is a quite time consuming process, and it
will make user confuse that what the development is waiting for, so we add a
new status - Pulling to indicate the pending is caused by pulling operation.
### What type of PR is it?
Improvement.
### Todos
N/A
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-964
### How should this be tested?
Observe the status in notebook page after creating a notebook.
### Screenshots (if appropriate)
<img width="1346" alt="ζͺε 2021-09-01 δΈε6 44 52"
src="https://user-images.githubusercontent.com/5687317/131663293-2e7a0e60-6569-4379-bea2-f6f585fb9fb7.png";>
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: Brandon Lin <[email protected]>
Signed-off-by: Kevin <[email protected]>
Closes #735 from FatalLin/SUBMARINE-964 and squashes the following commits:
7b9eab1b [Brandon Lin] polish and correct typo
f95373bd [Brandon Lin] code polish
d28bedd0 [Brandon Lin] leverge labelselector to find pods specifically
6df144b2 [Brandon Lin] remove unused import
408d3ab4 [Brandon Lin] add new status pulling for notebook
---
helm-charts/submarine/templates/rbac.yaml | 1 +
.../submarine/server/api/notebook/Notebook.java | 3 ++-
.../submarine/server/api/spec/NotebookMeta.java | 19 +++++++++++++++
.../submarine/server/notebook/NotebookManager.java | 19 +++++++++++++++
.../server/submitter/k8s/K8sSubmitter.java | 28 ++++++++++++++++++----
.../server/submitter/k8s/model/NotebookCR.java | 3 ++-
.../submitter/k8s/parser/NotebookSpecParser.java | 1 +
.../submitter/k8s/NotebookSpecParserTest.java | 2 +-
8 files changed, 69 insertions(+), 7 deletions(-)
diff --git a/helm-charts/submarine/templates/rbac.yaml
b/helm-charts/submarine/templates/rbac.yaml
index fce2ef6..16e4c7e 100644
--- a/helm-charts/submarine/templates/rbac.yaml
+++ b/helm-charts/submarine/templates/rbac.yaml
@@ -59,6 +59,7 @@ rules:
- pods/log
- services
- persistentvolumeclaims
+ - events
verbs:
- '*'
- apiGroups:
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/Notebook.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/Notebook.java
index 1902cca..db8d055 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/Notebook.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/Notebook.java
@@ -111,7 +111,8 @@ public class Notebook {
STATUS_CREATING("creating"),
STATUS_RUNNING("running"),
STATUS_WAITING("waiting"),
- STATUS_TERMINATING("terminating");
+ STATUS_TERMINATING("terminating"),
+ STATUS_PULLING("pulling");
private String value;
Status(String value) {
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookMeta.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookMeta.java
index fc46a7f..2203aef 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookMeta.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/NotebookMeta.java
@@ -19,10 +19,13 @@
package org.apache.submarine.server.api.spec;
+import java.util.Map;
+
public class NotebookMeta {
private String name;
private String namespace;
private String ownerId;
+ private Map<String, String> labels;
public NotebookMeta() {
@@ -75,4 +78,20 @@ public class NotebookMeta {
public void setOwnerId(String ownerId) {
this.ownerId = ownerId;
}
+
+ /**
+ * Set the labels on Notebook
+ * @param Map labels
+ */
+ public Map<String, String> getLabels() {
+ return labels;
+ }
+ /**
+ * get labels on Notebook
+ * @return labels
+ */
+ public void setLabels(Map<String, String> labels) {
+ this.labels = labels;
+ }
+
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
index 3de8151..43c848b 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
@@ -31,7 +31,9 @@ import
org.apache.submarine.server.environment.EnvironmentManager;
import javax.ws.rs.core.Response;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.submarine.server.notebook.database.NotebookService;
import org.slf4j.Logger;
@@ -81,7 +83,17 @@ public class NotebookManager {
String lowerName = spec.getMeta().getName().toLowerCase();
spec.getMeta().setName(lowerName);
NotebookId notebookId = generateNotebookId();
+
+ Map<String, String> labels = spec.getMeta().getLabels();
+
+ if (labels == null) {
+ labels = new HashMap<>();
+ }
+ labels.put("notebook-owner-id", spec.getMeta().getOwnerId());
+ labels.put("notebook-id", notebookId.toString());
+ spec.getMeta().setLabels(labels);
Notebook notebook = submitter.createNotebook(spec);
+
notebook.setNotebookId(notebookId);
notebook.setSpec(spec);
@@ -198,6 +210,13 @@ public class NotebookManager {
throw new SubmarineRuntimeException(Response.Status.OK.getStatusCode(),
"Invalid. Notebook Spec object is null.");
}
+ List<Notebook> serviceNotebooks = notebookService.selectAll();
+ for (Notebook notebook: serviceNotebooks) {
+ if
(notebook.getSpec().getMeta().getName().equals(spec.getMeta().getName())) {
+ throw new SubmarineRuntimeException(Response.Status.OK.getStatusCode(),
+ "Invalid. Notebook with same name is already existed.");
+ }
+ }
}
private void checkNotebookId(String id) throws SubmarineRuntimeException {
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
index ddc76af..0a23e58 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
@@ -38,6 +38,8 @@ import io.kubernetes.client.apis.CoreV1Api;
import io.kubernetes.client.apis.CustomObjectsApi;
import io.kubernetes.client.models.V1DeleteOptionsBuilder;
import io.kubernetes.client.models.V1Deployment;
+import io.kubernetes.client.models.V1Event;
+import io.kubernetes.client.models.V1EventList;
import io.kubernetes.client.models.V1ObjectMeta;
import io.kubernetes.client.models.V1PersistentVolumeClaim;
import io.kubernetes.client.models.V1Pod;
@@ -371,9 +373,7 @@ public class K8sSubmitter implements Submitter {
NotebookCR notebookCR;
try {
notebookCR = NotebookSpecParser.parseNotebook(spec);
- Map<String, String> labels = new HashMap<>();
- labels.put(NotebookCR.NOTEBOOK_OWNER_SELECTOR_KET,
spec.getMeta().getOwnerId());
- notebookCR.getMetadata().setLabels(labels);
+
notebookCR.getMetadata().setNamespace(namespace);
notebookCR.getMetadata().setOwnerReferences(OwnerReferenceUtils.getOwnerReference());
} catch (JsonSyntaxException e) {
@@ -429,10 +429,30 @@ public class K8sSubmitter implements Submitter {
try {
NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
+
Object object = api.getNamespacedCustomObject(notebookCR.getGroup(),
notebookCR.getVersion(),
namespace,
notebookCR.getPlural(), notebookCR.getMetadata().getName());
notebook = NotebookUtils.parseObject(object,
NotebookUtils.ParseOpt.PARSE_OPT_GET);
+ if
(notebook.getStatus().equals(Notebook.Status.STATUS_WAITING.toString())) {
+ LOG.info(String.format("notebook status: waiting; check the pods in
namespace:[%s] to "
+ + "ensure is the waiting caused by image pulling", namespace));
+ String podLabelSelector = String.format("%s=%s",
NotebookCR.NOTEBOOK_ID,
+ spec.getMeta().getLabels().get(NotebookCR.NOTEBOOK_ID).toString());
+
+ V1PodList podList = coreApi.listNamespacedPod(namespace, null, null,
null, podLabelSelector,
+ null, null, null, null);
+ String podName = podList.getItems().get(0).getMetadata().getName();
+ String fieldSelector = String.format("involvedObject.name=%s",
podName);
+ V1EventList events = coreApi.listNamespacedEvent(namespace, null,
null, fieldSelector,
+ null, null, null, null, null);
+ V1Event latestEvent = events.getItems().get(events.getItems().size() -
1);
+
+ if (latestEvent.getReason().equalsIgnoreCase("Pulling")) {
+ notebook.setStatus(Notebook.Status.STATUS_PULLING.getValue());
+ notebook.setReason(latestEvent.getReason());
+ }
+ }
} catch (ApiException e) {
throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
}
@@ -470,7 +490,7 @@ public class K8sSubmitter implements Submitter {
try {
Object object =
api.listNamespacedCustomObject(NotebookCR.CRD_NOTEBOOK_GROUP_V1,
NotebookCR.CRD_NOTEBOOK_VERSION_V1, namespace,
NotebookCR.CRD_NOTEBOOK_PLURAL_V1,
- "true", null, NotebookCR.NOTEBOOK_OWNER_SELECTOR_KET + "=" + id,
+ "true", null, NotebookCR.NOTEBOOK_OWNER_SELECTOR_KEY + "=" + id,
null, null, null);
notebookList = NotebookUtils.parseObjectForList(object);
} catch (ApiException e) {
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
index c93e189..c77a1f2 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
@@ -29,7 +29,8 @@ public class NotebookCR {
public static final String CRD_APIVERSION_V1 = CRD_NOTEBOOK_GROUP_V1 + "/" +
CRD_NOTEBOOK_VERSION_V1;
public static final String CRD_NOTEBOOK_KIND_V1 = "Notebook";
public static final String CRD_NOTEBOOK_PLURAL_V1 = "notebooks";
- public static final String NOTEBOOK_OWNER_SELECTOR_KET = "notebook-owner-id";
+ public static final String NOTEBOOK_OWNER_SELECTOR_KEY = "notebook-owner-id";
+ public static final String NOTEBOOK_ID = "notebook-id";
@SerializedName("apiVersion")
private String apiVersion;
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
index d03138f..c5f588c 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
@@ -64,6 +64,7 @@ public class NotebookSpecParser {
V1ObjectMeta meta = new V1ObjectMeta();
meta.setName(spec.getMeta().getName());
meta.setNamespace(spec.getMeta().getNamespace());
+ meta.setLabels(spec.getMeta().getLabels());
return meta;
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
index e982a25..a0d680b 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
@@ -50,7 +50,7 @@ public class NotebookSpecParserTest extends SpecBuilder {
Assert.assertEquals(meta.getName(), actualMeta.getName());
Assert.assertEquals(meta.getNamespace(), actualMeta.getNamespace());
Assert.assertEquals(meta.getOwnerId(),
-
actualMeta.getLabels().get(NotebookCR.NOTEBOOK_OWNER_SELECTOR_KET));
+
actualMeta.getLabels().get(NotebookCR.NOTEBOOK_OWNER_SELECTOR_KEY));
}
private void validateEnvironment(NotebookSpec spec, NotebookCRSpec
actualPodSpec) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
![https://user-images.githubusercontent.com/5687317/131663293-2e7a0e60-6569-4379-bea2-f6f585fb9fb7.png"](https://user-images.githubusercontent.com/5687317/131663293-2e7a0e60-6569-4379-bea2-f6f585fb9fb7.png"){kind=link}