This is an automated email from the ASF dual-hosted git repository.
kuanhsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 21da776 SUBMARINE-1218. delete agent pod as well both in the deleting
operation of experiment and notebook
21da776 is described below
commit 21da776810b20b509fb2f8443b4af29ad391f8e3
Author: FatalLin <[email protected]>
AuthorDate: Thu Mar 17 23:19:20 2022 +0800
SUBMARINE-1218. delete agent pod as well both in the deleting operation of
experiment and notebook
### What is this PR for?
Just like KUAN-HSUN-LI mentioned in jira ticket, the existed agent pod
will occurs error if we recreate the custom resource with the same name with
deleing resource, so I fired this PR to add the step to delete agent pod in the
deleting operation.
### What type of PR is it?
Bug Fix
### Todos
N/A
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-1218
### How should this be tested?
should pass existed tests.
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating?No
* Are there breaking changes for older versions?No
* Does this need new documentation?No
Author: FatalLin <[email protected]>
Signed-off-by: kuanhsun <[email protected]>
Closes #904 from FatalLin/SUBMARINE-1218 and squashes the following commits:
4b1cc577 [FatalLin] delete agent pod as well both in the deleting operation
of experiment and notebook
---
.../org/apache/submarine/server/api/Submitter.java | 3 ++-
.../submarine/server/notebook/NotebookManager.java | 2 +-
.../server/submitter/k8s/K8sSubmitter.java | 24 ++++++++++++++++++----
.../submitter/k8s/parser/NotebookSpecParser.java | 2 +-
.../submitter/k8s/NotebookSpecParserTest.java | 2 +-
5 files changed, 25 insertions(+), 8 deletions(-)
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
index 2f8706e..7f3a480 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
@@ -111,10 +111,11 @@ public interface Submitter {
/**
* Delete a notebook with spec
* @param spec spec
+ * @param notebookId notebookId
* @return object
* @throws SubmarineRuntimeException running error
*/
- Notebook deleteNotebook(NotebookSpec spec) throws SubmarineRuntimeException;
+ Notebook deleteNotebook(NotebookSpec spec, String notebookId) throws
SubmarineRuntimeException;
/**
* List notebooks with userID
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
index dcdfbdf..5009b05 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java
@@ -172,7 +172,7 @@ public class NotebookManager {
*/
public Notebook deleteNotebook(String id) throws SubmarineRuntimeException {
Notebook notebook = getNotebook(id);
- Notebook patchNotebook = submitter.deleteNotebook(notebook.getSpec());
+ Notebook patchNotebook = submitter.deleteNotebook(notebook.getSpec(), id);
notebookService.delete(id);
notebook.rebuild(patchNotebook);
return notebook;
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
index 5e961fc..2229e15 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
@@ -341,6 +341,12 @@ public class K8sSubmitter implements Submitter {
try {
MLJob mlJob = ExperimentSpecParser.parseJob(spec);
mlJob.getMetadata().setNamespace(getServerNamespace());
+
+ AgentPod agentPod = new AgentPod(getServerNamespace(),
spec.getMeta().getName(),
+ mlJob.getPlural().equals(TFJob.CRD_TF_PLURAL_V1)
+ ? CustomResourceType.TFJob : CustomResourceType.PyTorchJob,
+ spec.getMeta().getExperimentId());
+
Object object = mlJob.getPlural().equals(TFJob.CRD_TF_PLURAL_V1)
? tfJobClient.delete(getServerNamespace(),
mlJob.getMetadata().getName(),
MLJobConverter.toDeleteOptionsFromMLJob(mlJob))
@@ -348,6 +354,10 @@ public class K8sSubmitter implements Submitter {
: pyTorchJobClient.delete(getServerNamespace(),
mlJob.getMetadata().getName(),
MLJobConverter.toDeleteOptionsFromMLJob(mlJob))
.throwsApiException().getStatus();
+
+ LOG.info(String.format("Experiment:%s had been deleted, start to delete
agent pod:%s",
+ spec.getMeta().getName(), agentPod.getMetadata().getName()));
+ podClient.delete(agentPod.getMetadata().getNamespace(),
agentPod.getMetadata().getName());
experiment = parseExperimentResponseObject(object,
ParseOp.PARSE_OP_DELETE);
} catch (InvalidSpecException e) {
throw new SubmarineRuntimeException(200, e.getMessage());
@@ -475,7 +485,7 @@ public class K8sSubmitter implements Submitter {
// parse notebook custom resource
NotebookCR notebookCR;
try {
- notebookCR = NotebookSpecParser.parseNotebook(spec, notebookId,
namespace);
+ notebookCR = NotebookSpecParser.parseNotebook(spec, namespace);
notebookCR.getMetadata().setNamespace(namespace);
notebookCR.getMetadata().setOwnerReferences(OwnerReferenceUtils.getOwnerReference());
} catch (JsonSyntaxException e) {
@@ -553,7 +563,7 @@ public class K8sSubmitter implements Submitter {
String namespace = getServerNamespace();
try {
- NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec, null,
null);
+ NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec, null);
Object object = notebookCRClient.get(namespace,
notebookCR.getMetadata().getName())
.throwsApiException().getObject();
notebook = NotebookUtils.parseObject(object,
NotebookUtils.ParseOpt.PARSE_OPT_GET);
@@ -575,11 +585,14 @@ public class K8sSubmitter implements Submitter {
}
@Override
- public Notebook deleteNotebook(NotebookSpec spec) throws
SubmarineRuntimeException {
+ public Notebook deleteNotebook(NotebookSpec spec, String notebookId) throws
SubmarineRuntimeException {
Notebook notebook = null;
final String name = spec.getMeta().getName();
String namespace = getServerNamespace();
- NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec, null, null);
+ NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec, null);
+ AgentPod agentPod = new AgentPod(namespace, spec.getMeta().getName(),
+ CustomResourceType.Notebook, notebookId);
+
try {
Object object = notebookCRClient.delete(namespace, name,
getDeleteOptions(notebookCR.getApiVersion())).throwsApiException().getStatus();
@@ -610,6 +623,9 @@ public class K8sSubmitter implements Submitter {
if (StringUtils.isNoneBlank(OVERWRITE_JSON)) {
deleteConfigMap(namespace, String.format("%s-%s",
NotebookUtils.OVERWRITE_PREFIX, name));
}
+ LOG.info(String.format("Experiment:%s had been deleted, start to delete
agent pod:%s",
+ spec.getMeta().getName(), agentPod.getMetadata().getName()));
+ podClient.delete(agentPod.getMetadata().getNamespace(),
agentPod.getMetadata().getName());
return notebook;
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
index 3bcd09e..5783771 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
@@ -63,7 +63,7 @@ public class NotebookSpecParser {
SubmarineConfiguration.getInstance();
- public static NotebookCR parseNotebook(NotebookSpec spec, String notebookId,
String namespace) {
+ public static NotebookCR parseNotebook(NotebookSpec spec, String namespace) {
NotebookCR notebookCR = new NotebookCR();
notebookCR.setMetadata(parseMetadata(spec));
notebookCR.setSpec(parseNotebookCRSpec(spec));
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
index d493f23..c2533c8 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
@@ -41,7 +41,7 @@ public class NotebookSpecParserTest extends SpecBuilder {
@Test
public void testValidNotebook() throws IOException, URISyntaxException {
NotebookSpec notebookSpec = (NotebookSpec)
buildFromJsonFile(NotebookSpec.class, notebookReqFile);
- NotebookCR notebook = NotebookSpecParser.parseNotebook(notebookSpec, null,
null);
+ NotebookCR notebook = NotebookSpecParser.parseNotebook(notebookSpec, null);
validateMetadata(notebookSpec.getMeta(), notebook.getMetadata());
validateEnvironment(notebookSpec, notebook.getSpec());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
