This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 72be805 SUBMARINE-851. Dirty notebook status is persisted in DB when
notebook operation is failed
72be805 is described below
commit 72be805fc4672f7f29c96cca95e16d59358d4dd2
Author: Lisa <[email protected]>
AuthorDate: Thu Jul 15 11:11:42 2021 +0800
SUBMARINE-851. Dirty notebook status is persisted in DB when notebook
operation is failed
### What is this PR for?
Maintain atomicity while creating notebook.
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-851
### How should this be tested?
Try to get error in notebook creation, and then create a new notebook with
same name.
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: Lisa <[email protected]>
Signed-off-by: Kevin <[email protected]>
Closes #655 from aeioulisa/SUBMARINE-851 and squashes the following commits:
6c3634a7 [Lisa] edit function name
6c637a4f [Lisa] edit function name
bf6285a4 [Lisa] edit log and try-block
3916827e [Lisa] Let notebook creation maintain Atomicity
---
.../server/submitter/k8s/K8sSubmitter.java | 78 +++++++++++++++++-----
1 file changed, 61 insertions(+), 17 deletions(-)
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
index 0e7accf..a530d16 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
@@ -88,7 +88,7 @@ public class K8sSubmitter implements Submitter {
private static final String TF_JOB_SELECTOR_KEY = "tf-job-name=";
private static final String PYTORCH_JOB_SELECTOR_KEY = "pytorch-job-name=";
-
+
private static final String ENV_NAMESPACE = "ENV_NAMESPACE";
// K8s API client for CRD
@@ -361,32 +361,50 @@ public class K8sSubmitter implements Submitter {
final String storage = NotebookUtils.STORAGE;
final String pvcName = NotebookUtils.PVC_PREFIX + name;
String namespace = "default";
-
+ NotebookCR notebookCR;
+
if (System.getenv(ENV_NAMESPACE) != null) {
namespace = System.getenv(ENV_NAMESPACE);
}
-
+
try {
// create notebook custom resource
- NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
+ notebookCR = NotebookSpecParser.parseNotebook(spec);
Map<String, String> labels = new HashMap<>();
labels.put(NotebookCR.NOTEBOOK_OWNER_SELECTOR_KET,
spec.getMeta().getOwnerId());
notebookCR.getMetadata().setLabels(labels);
notebookCR.getMetadata().setNamespace(namespace);
-
+
// create persistent volume
createPersistentVolume(pvName, host, storage);
+ } catch (ApiException e) {
+ LOG.error("K8s submitter: Create persistent volume for Notebook object
failed by " + e.getMessage(), e);
+ throw new SubmarineRuntimeException(e.getCode(), "K8s submitter: Create
persistent volume for " +
+ "Notebook object failed by " + e.getMessage());
+ } catch (JsonSyntaxException e) {
+ LOG.error("K8s submitter: parse response object failed by " +
e.getMessage(), e);
+ throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream
response failed.");
+ }
- // create persistent volume claim
+ // create persistent volume claim
+ try {
createPersistentVolumeClaim(pvcName, namespace, pvName, storage);
+ } catch (ApiException e) {
+ LOG.error("K8s submitter: Create persistent volume claim for Notebook
object failed by " +
+ e.getMessage(), e);
+ rollbackCreationPV(pvName);
+ throw new SubmarineRuntimeException(e.getCode(), "K8s submitter: Create
persistent volume claim for " +
+ "Notebook object failed by " + e.getMessage());
+ }
- // bind persistent volume claim
+ // bind persistent volume claim
+ try {
V1PersistentVolumeClaimVolumeSource pvcSource = new
V1PersistentVolumeClaimVolumeSource()
- .claimName(pvcName);
+ .claimName(pvcName);
notebookCR.getSpec().getTemplate().getSpec().getVolumes().get(0).persistentVolumeClaim(pvcSource);
Object object = api.createNamespacedCustomObject(notebookCR.getGroup(),
notebookCR.getVersion(),
- namespace, notebookCR.getPlural(), notebookCR, "true");
+ namespace, notebookCR.getPlural(), notebookCR, "true");
notebook = NotebookUtils.parseObject(object,
NotebookUtils.ParseOpt.PARSE_OPT_CREATE);
// create Traefik custom resource
@@ -394,9 +412,11 @@ public class K8sSubmitter implements Submitter {
} catch (JsonSyntaxException e) {
LOG.error("K8s submitter: parse response object failed by " +
e.getMessage(), e);
+ rollbackCreationPVC(pvName, pvcName, namespace);
throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream
response failed.");
} catch (ApiException e) {
LOG.error("K8s submitter: parse Notebook object failed by " +
e.getMessage(), e);
+ rollbackCreationPVC(pvName, pvcName, namespace);
throw new SubmarineRuntimeException(e.getCode(), "K8s submitter: parse
Notebook object failed by " +
e.getMessage());
}
@@ -407,12 +427,12 @@ public class K8sSubmitter implements Submitter {
public Notebook findNotebook(NotebookSpec spec) throws
SubmarineRuntimeException {
Notebook notebook;
String namespace = "default";
-
+
if (System.getenv(ENV_NAMESPACE) != null) {
namespace = System.getenv(ENV_NAMESPACE);
}
-
- try {
+
+ try {
NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
Object object = api.getNamespacedCustomObject(notebookCR.getGroup(),
notebookCR.getVersion(),
namespace,
@@ -431,11 +451,11 @@ public class K8sSubmitter implements Submitter {
final String pvName = NotebookUtils.PV_PREFIX + name;
final String pvcName = NotebookUtils.PVC_PREFIX + name;
String namespace = "default";
-
+
if (System.getenv(ENV_NAMESPACE) != null) {
namespace = System.getenv(ENV_NAMESPACE);
}
-
+
try {
NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
Object object = api.deleteNamespacedCustomObject(notebookCR.getGroup(),
notebookCR.getVersion(),
@@ -469,7 +489,7 @@ public class K8sSubmitter implements Submitter {
}
@Override
- public ServeResponse createServe(ServeRequest spec)
+ public ServeResponse createServe(ServeRequest spec)
throws SubmarineRuntimeException {
String modelName = spec.getModelName();
String modelVersion = spec.getModelVersion();
@@ -490,7 +510,7 @@ public class K8sSubmitter implements Submitter {
middleware.getGroup(), middleware.getVersion(),
middleware.getMetadata().getNamespace(),
middleware.getPlural(), middleware, "true");
-
+
api.createNamespacedCustomObject(
ingressRoute.getGroup(), ingressRoute.getVersion(),
ingressRoute.getMetadata().getNamespace(),
@@ -502,7 +522,7 @@ public class K8sSubmitter implements Submitter {
}
@Override
- public ServeResponse deleteServe(ServeRequest spec)
+ public ServeResponse deleteServe(ServeRequest spec)
throws SubmarineRuntimeException {
String modelName = spec.getModelName();
String modelVersion = spec.getModelVersion();
@@ -683,6 +703,30 @@ public class K8sSubmitter implements Submitter {
return spec;
}
+ private void rollbackCreationPV(String pvName) {
+ try {
+ deletePersistentVolume(pvName);
+ } catch (ApiException e) {
+ LOG.error("K8s submitter: delete persistent volume failed by {}, may
cause some dirty data",
+ e.getMessage());
+ }
+ }
+
+ private void rollbackCreationPVC(String pvName, String pvcName, String
namespace) {
+ try {
+ deletePersistentVolumeClaim(pvcName, namespace);
+ } catch (ApiException e) {
+ LOG.error("K8s submitter: delete persistent volume claim failed by {},
may cause some dirty data",
+ e.getMessage());
+ }
+ try {
+ deletePersistentVolume(pvName);
+ } catch (ApiException e) {
+ LOG.error("K8s submitter: delete persistent volume failed by {}, may
cause some dirty data",
+ e.getMessage());
+ }
+ }
+
private enum ParseOp {
PARSE_OP_RESULT,
PARSE_OP_DELETE
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
