This is an automated email from the ASF dual-hosted git repository.

kuanhsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git


The following commit(s) were added to refs/heads/master by this push:
     new 50f21b5  SUBMARINE-1109. Java k8s client version from 6 upgrade to 11
50f21b5 is described below

commit 50f21b5b1e1ab986f1aaaaa0eb3633de97ae14df
Author: noidname01 <[email protected]>
AuthorDate: Sun Dec 19 22:26:17 2021 +0800

    SUBMARINE-1109. Java k8s client version from 6 upgrade to 11
    
    ### What is this PR for?
    The version of k8s client we use is almost out of date, and lacks many 
features, like CustomResources Management, Generic Api and more concise 
function.
    
    ### What type of PR is it?
    [Improvement]
    
    ### Todos
    
    None
    
    ### What is the Jira issue?
    
    SUBMARINE-1109. Java k8s client version from 6 upgrade to 11
    
    ### How should this be tested?
    
    None
    
    ### Screenshots (if appropriate)
    
    None
    
    ### Questions:
    * Do the license files need updating? No
    * Are there breaking changes for older versions? No
    * Does this need new documentation? No
    
    Author: noidname01 <[email protected]>
    
    Signed-off-by: kuanhsun <[email protected]>
    
    Closes #820 from noidname01/SUBMARINE-1109 and squashes the following 
commits:
    
    68ad9bb5 [noidname01] checkstyle
    f6bdb9c3 [noidname01] final
    7ecf58e3 [noidname01] fix rebasing
    551c1a33 [noidname01] fix rebase error
    ee3fb277 [noidname01] delete comment
    e2d2a9c6 [noidname01] delete some foolish
    5b1abfaa [noidname01] remove redundant import
    6fde016f [noidname01] modify import
    fe1c223e [noidname01] modify import name
    5d7949e0 [noidname01] WIP
---
 pom.xml                                            |   6 +-
 .../submarine/serve/istio/IstioVirtualService.java |   2 +-
 .../serve/pytorch/SeldonPytorchServing.java        |   2 +-
 .../submarine/serve/seldon/SeldonDeployment.java   |   2 +-
 .../serve/tensorflow/SeldonTFServing.java          |   2 +-
 .../server/submitter/k8s/K8sSubmitter.java         | 119 ++++++++++-----------
 .../codelocalizer/AbstractCodeLocalizer.java       |  12 +--
 .../experiment/codelocalizer/CodeLocalizer.java    |   2 +-
 .../codelocalizer/DummyCodeLocalizer.java          |   2 +-
 .../experiment/codelocalizer/GitCodeLocalizer.java |   8 +-
 .../codelocalizer/HTTPGitCodeLocalizer.java        |   4 +-
 .../codelocalizer/SSHGitCodeLocalizer.java         |  16 +--
 .../server/submitter/k8s/model/ListMeta.java       |   2 +-
 .../server/submitter/k8s/model/MLJob.java          |   4 +-
 .../submitter/k8s/model/MLJobReplicaSpec.java      |   2 +-
 .../server/submitter/k8s/model/NotebookCR.java     |   2 +-
 .../server/submitter/k8s/model/NotebookCRSpec.java |   4 +-
 .../server/submitter/k8s/model/NotebookStatus.java |   2 +-
 .../k8s/model/ingressroute/IngressRoute.java       |   2 +-
 .../k8s/model/middlewares/Middlewares.java         |   2 +-
 .../submitter/k8s/parser/ExperimentSpecParser.java |  22 ++--
 .../submitter/k8s/parser/NotebookSpecParser.java   |  18 ++--
 .../submitter/k8s/parser/ServeSpecParser.java      |  26 ++---
 .../submitter/k8s/parser/VolumeSpecParser.java     |  14 +--
 .../server/submitter/k8s/util/MLJobConverter.java  |  12 +--
 .../server/submitter/k8s/util/NotebookUtils.java   |   6 +-
 .../submitter/k8s/util/OwnerReferenceUtils.java    |   2 +-
 .../submitter/k8s/ExperimentSpecParserTest.java    |  10 +-
 .../server/submitter/k8s/K8SJobSubmitterTest.java  |   2 +-
 .../server/submitter/k8s/MLJobConverterTest.java   |  14 +--
 .../submitter/k8s/NotebookSpecParserTest.java      |   4 +-
 .../apache/submarine/rest/ExperimentRestApiIT.java |  10 +-
 .../apache/submarine/rest/NotebookRestApiIT.java   |  10 +-
 33 files changed, 172 insertions(+), 175 deletions(-)

diff --git a/pom.xml b/pom.xml
index 333f269..26a4d84 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,7 +67,7 @@
 
     <!--library versions-->
     <tony.version>0.4.7</tony.version>
-    <jersey.version>2.27</jersey.version>
+    <jersey.version>2.35</jersey.version>
     <jersey.servlet.version>1.13</jersey.servlet.version>
     <jetty.version>9.4.35.v20201120</jetty.version>
     <javax.ws.rsapi.version>2.1</javax.ws.rsapi.version>
@@ -127,10 +127,10 @@
     <mybatis-generator.version>1.3.7</mybatis-generator.version>
     <derby.version>10.15.1.3</derby.version>
     <zeppelin.version>0.9.0-preview1</zeppelin.version>
-    <jgit.version>5.5.1.201910021850-r</jgit.version>
+    <jgit.version>5.13.0.202109080827-r</jgit.version>
     <atomix.version>3.1.5</atomix.version>
     <!--  Submarine on Kubernetes  -->
-    <k8s.client-java.version>6.0.1</k8s.client-java.version>
+    <k8s.client-java.version>11.0.1</k8s.client-java.version>
     <jersey.test-framework>2.27</jersey.test-framework>
     <!-- integration test-->
     <plugin.failsafe.version>2.17</plugin.failsafe.version>
diff --git 
a/submarine-serve/src/main/java/org/apache/submarine/serve/istio/IstioVirtualService.java
 
b/submarine-serve/src/main/java/org/apache/submarine/serve/istio/IstioVirtualService.java
index dbe90de..955e112 100644
--- 
a/submarine-serve/src/main/java/org/apache/submarine/serve/istio/IstioVirtualService.java
+++ 
b/submarine-serve/src/main/java/org/apache/submarine/serve/istio/IstioVirtualService.java
@@ -19,7 +19,7 @@
 package org.apache.submarine.serve.istio;
 
 import com.google.gson.annotations.SerializedName;
-import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import org.apache.submarine.serve.utils.IstioConstants;
 
 import java.util.ArrayList;
diff --git 
a/submarine-serve/src/main/java/org/apache/submarine/serve/pytorch/SeldonPytorchServing.java
 
b/submarine-serve/src/main/java/org/apache/submarine/serve/pytorch/SeldonPytorchServing.java
index b7881d6..3993ee7 100644
--- 
a/submarine-serve/src/main/java/org/apache/submarine/serve/pytorch/SeldonPytorchServing.java
+++ 
b/submarine-serve/src/main/java/org/apache/submarine/serve/pytorch/SeldonPytorchServing.java
@@ -18,7 +18,7 @@
  */
 package org.apache.submarine.serve.pytorch;
 
-import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import org.apache.submarine.serve.seldon.SeldonDeployment;
 import org.apache.submarine.serve.seldon.SeldonGraph;
 import org.apache.submarine.serve.seldon.SeldonPredictor;
diff --git 
a/submarine-serve/src/main/java/org/apache/submarine/serve/seldon/SeldonDeployment.java
 
b/submarine-serve/src/main/java/org/apache/submarine/serve/seldon/SeldonDeployment.java
index 483d24c..30c29dd 100644
--- 
a/submarine-serve/src/main/java/org/apache/submarine/serve/seldon/SeldonDeployment.java
+++ 
b/submarine-serve/src/main/java/org/apache/submarine/serve/seldon/SeldonDeployment.java
@@ -19,7 +19,7 @@
 package org.apache.submarine.serve.seldon;
 
 import com.google.gson.annotations.SerializedName;
-import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import org.apache.submarine.serve.utils.SeldonConstants;
 
 import java.util.ArrayList;
diff --git 
a/submarine-serve/src/main/java/org/apache/submarine/serve/tensorflow/SeldonTFServing.java
 
b/submarine-serve/src/main/java/org/apache/submarine/serve/tensorflow/SeldonTFServing.java
index d63dbaa..91565fd 100644
--- 
a/submarine-serve/src/main/java/org/apache/submarine/serve/tensorflow/SeldonTFServing.java
+++ 
b/submarine-serve/src/main/java/org/apache/submarine/serve/tensorflow/SeldonTFServing.java
@@ -18,7 +18,7 @@
  */
 package org.apache.submarine.serve.tensorflow;
 
-import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import org.apache.submarine.serve.seldon.SeldonDeployment;
 import org.apache.submarine.serve.seldon.SeldonGraph;
 import org.apache.submarine.serve.seldon.SeldonPredictor;
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
index 131e927..1befeca 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
@@ -28,32 +28,31 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
 
 
 import com.google.common.reflect.TypeToken;
 import com.google.gson.Gson;
 import com.google.gson.JsonSyntaxException;
-import com.squareup.okhttp.OkHttpClient;
-import io.kubernetes.client.ApiClient;
-import io.kubernetes.client.ApiException;
-import io.kubernetes.client.Configuration;
-import io.kubernetes.client.JSON;
-import io.kubernetes.client.apis.AppsV1Api;
-import io.kubernetes.client.apis.CoreV1Api;
-import io.kubernetes.client.apis.CustomObjectsApi;
-import io.kubernetes.client.models.V1DeleteOptionsBuilder;
-import io.kubernetes.client.models.V1Deployment;
-import io.kubernetes.client.models.V1Event;
-import io.kubernetes.client.models.V1EventList;
-import io.kubernetes.client.models.V1ObjectMeta;
-import io.kubernetes.client.models.V1PersistentVolumeClaim;
-import io.kubernetes.client.models.V1Pod;
-import io.kubernetes.client.models.V1PodList;
-import io.kubernetes.client.models.V1Status;
+import okhttp3.OkHttpClient;
+import io.kubernetes.client.openapi.ApiClient;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.Configuration;
+import io.kubernetes.client.openapi.JSON;
+import io.kubernetes.client.openapi.apis.AppsV1Api;
+import io.kubernetes.client.openapi.apis.CoreV1Api;
+import io.kubernetes.client.openapi.apis.CustomObjectsApi;
+import io.kubernetes.client.openapi.models.V1DeleteOptionsBuilder;
+import io.kubernetes.client.openapi.models.V1Deployment;
+import io.kubernetes.client.openapi.models.CoreV1Event;
+import io.kubernetes.client.openapi.models.CoreV1EventList;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim;
+import io.kubernetes.client.openapi.models.V1Pod;
+import io.kubernetes.client.openapi.models.V1PodList;
+import io.kubernetes.client.openapi.models.V1Status;
+import io.kubernetes.client.util.Watch;
 import io.kubernetes.client.util.ClientBuilder;
 import io.kubernetes.client.util.KubeConfig;
-import io.kubernetes.client.util.Watch;
 
 import org.apache.submarine.commons.utils.SubmarineConfiguration;
 import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
@@ -130,8 +129,8 @@ public class K8sSubmitter implements Submitter {
       }
     } finally {
       // let watcher can wait until the next change
+      client.setReadTimeout(0);
       OkHttpClient httpClient = client.getHttpClient();
-      httpClient.setReadTimeout(0, TimeUnit.SECONDS);
       client.setHttpClient(httpClient);
       Configuration.setDefaultApiClient(client);
     }
@@ -164,7 +163,7 @@ public class K8sSubmitter implements Submitter {
       
mlJob.getMetadata().setOwnerReferences(OwnerReferenceUtils.getOwnerReference());
 
       Object object = api.createNamespacedCustomObject(mlJob.getGroup(), 
mlJob.getVersion(),
-          mlJob.getMetadata().getNamespace(), mlJob.getPlural(), mlJob, 
"true");
+          mlJob.getMetadata().getNamespace(), mlJob.getPlural(), mlJob, 
"true", null, null);
       experiment = parseExperimentResponseObject(object, 
ParseOp.PARSE_OP_RESULT);
     } catch (InvalidSpecException e) {
       LOG.error("K8s submitter: parse Job object failed by " + e.getMessage(), 
e);
@@ -206,7 +205,7 @@ public class K8sSubmitter implements Submitter {
 
       Object object = api.patchNamespacedCustomObject(mlJob.getGroup(), 
mlJob.getVersion(),
           mlJob.getMetadata().getNamespace(), mlJob.getPlural(), 
mlJob.getMetadata().getName(),
-          mlJob);
+          mlJob, null, null, false);
       experiment = parseExperimentResponseObject(object, 
ParseOp.PARSE_OP_RESULT);
     } catch (InvalidSpecException e) {
       throw new SubmarineRuntimeException(200, e.getMessage());
@@ -224,8 +223,8 @@ public class K8sSubmitter implements Submitter {
       mlJob.getMetadata().setNamespace(getServerNamespace());
 
       Object object = api.deleteNamespacedCustomObject(mlJob.getGroup(), 
mlJob.getVersion(),
-          mlJob.getMetadata().getNamespace(), mlJob.getPlural(), 
mlJob.getMetadata().getName(),
-          MLJobConverter.toDeleteOptionsFromMLJob(mlJob), null, null, null);
+          mlJob.getMetadata().getNamespace(), mlJob.getPlural(), 
mlJob.getMetadata().getName(), 0,
+              false, null, null, 
MLJobConverter.toDeleteOptionsFromMLJob(mlJob));
       experiment = parseExperimentResponseObject(object, 
ParseOp.PARSE_OP_DELETE);
     } catch (InvalidSpecException e) {
       throw new SubmarineRuntimeException(200, e.getMessage());
@@ -261,9 +260,9 @@ public class K8sSubmitter implements Submitter {
     try {
       final V1PodList podList = coreApi.listNamespacedPod(
           getServerNamespace(),
-          "false", null, null,
+          "false", false,  null, null,
           getJobLabelSelector(spec), null, null,
-          null, null);
+          null, null, null);
       for (V1Pod pod : podList.getItems()) {
         String podName = pod.getMetadata().getName();
         experimentLog.addPodLog(podName, null);
@@ -280,15 +279,15 @@ public class K8sSubmitter implements Submitter {
     experimentLog.setExperimentId(id);
     try {
       final V1PodList podList = coreApi.listNamespacedPod(
-          getServerNamespace(),
-          "false", null, null,
-          getJobLabelSelector(spec), null, null,
-          null, null);
+              getServerNamespace(),
+              "false", false,  null, null,
+              getJobLabelSelector(spec), null, null,
+              null, null, null);
 
       for (V1Pod pod : podList.getItems()) {
         String podName = pod.getMetadata().getName();
         String podLog = coreApi.readNamespacedPodLog(
-            podName, getServerNamespace(), null, Boolean.FALSE,
+            podName, getServerNamespace(), null, Boolean.FALSE, null,
             Integer.MAX_VALUE, null, Boolean.FALSE,
             Integer.MAX_VALUE, null, Boolean.FALSE);
 
@@ -413,7 +412,7 @@ public class K8sSubmitter implements Submitter {
     // create notebook custom resource
     try {
       Object object = api.createNamespacedCustomObject(notebookCR.getGroup(), 
notebookCR.getVersion(),
-          namespace, notebookCR.getPlural(), notebookCR, "true");
+          namespace, notebookCR.getPlural(), notebookCR, "true", null, null);
       notebook = NotebookUtils.parseObject(object, 
NotebookUtils.ParseOpt.PARSE_OPT_CREATE);
     } catch (JsonSyntaxException e) {
       LOG.error("K8s submitter: parse response object failed by " + 
e.getMessage(), e);
@@ -459,13 +458,13 @@ public class K8sSubmitter implements Submitter {
         String podLabelSelector = String.format("%s=%s", 
NotebookCR.NOTEBOOK_ID,
             spec.getMeta().getLabels().get(NotebookCR.NOTEBOOK_ID).toString());
 
-        V1PodList podList = coreApi.listNamespacedPod(namespace, null, null, 
null, podLabelSelector,
-            null, null, null, null);
+        V1PodList podList = coreApi.listNamespacedPod(namespace, null, null, 
null, null,
+                podLabelSelector, null, null, null, null, null);
         String podName = podList.getItems().get(0).getMetadata().getName();
         String fieldSelector = String.format("involvedObject.name=%s", 
podName);
-        V1EventList events = coreApi.listNamespacedEvent(namespace, null, 
null, fieldSelector,
-            null, null, null, null, null);
-        V1Event latestEvent = events.getItems().get(events.getItems().size() - 
1);
+        CoreV1EventList events = coreApi.listNamespacedEvent(namespace, null, 
null, null, fieldSelector,
+            null, null, null, null, null, null);
+        CoreV1Event latestEvent = 
events.getItems().get(events.getItems().size() - 1);
 
         if (latestEvent.getReason().equalsIgnoreCase("Pulling")) {
           notebook.setStatus(Notebook.Status.STATUS_PULLING.getValue());
@@ -489,9 +488,8 @@ public class K8sSubmitter implements Submitter {
       NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
       Object object = api.deleteNamespacedCustomObject(notebookCR.getGroup(), 
notebookCR.getVersion(),
           namespace, notebookCR.getPlural(),
-          notebookCR.getMetadata().getName(),
-          new 
V1DeleteOptionsBuilder().withApiVersion(notebookCR.getApiVersion()).build(),
-          null, null, null);
+          notebookCR.getMetadata().getName(), null, null, null,
+              null, new 
V1DeleteOptionsBuilder().withApiVersion(notebookCR.getApiVersion()).build());
       notebook = NotebookUtils.parseObject(object, 
NotebookUtils.ParseOpt.PARSE_OPT_DELETE);
       deleteIngressRoute(namespace, notebookCR.getMetadata().getName());
       deletePersistentVolumeClaim(pvcName, namespace);
@@ -509,8 +507,8 @@ public class K8sSubmitter implements Submitter {
     try {
       Object object = 
api.listNamespacedCustomObject(NotebookCR.CRD_NOTEBOOK_GROUP_V1,
           NotebookCR.CRD_NOTEBOOK_VERSION_V1, namespace, 
NotebookCR.CRD_NOTEBOOK_PLURAL_V1,
-          "true", null, NotebookCR.NOTEBOOK_OWNER_SELECTOR_KEY + "=" + id,
-          null, null, null);
+          "true", null, null, NotebookCR.NOTEBOOK_OWNER_SELECTOR_KEY + "=" + 
id,
+          null, null, null, null);
       notebookList = NotebookUtils.parseObjectForList(object);
     } catch (ApiException e) {
       throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
@@ -530,7 +528,7 @@ public class K8sSubmitter implements Submitter {
                "default",
                seldonDeployment.getPlural(),
                seldonDeployment,
-               "true");
+               "true", null, null);
     } catch (ApiException e) {
       LOG.error(e.getMessage(), e);
       throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
@@ -541,7 +539,7 @@ public class K8sSubmitter implements Submitter {
               "default",
               istioVirtualService.getPlural(),
               istioVirtualService,
-              "true");
+              "true", null, null);
     } catch (ApiException e) {
       LOG.error(e.getMessage(), e);
       try {
@@ -550,8 +548,8 @@ public class K8sSubmitter implements Submitter {
               "default",
               seldonDeployment.getPlural(),
               seldonDeployment.getMetadata().getName(),
-              new 
V1DeleteOptionsBuilder().withApiVersion(seldonDeployment.getApiVersion()).build(),
-              null, null, null);
+              null, null, null, null,
+              new 
V1DeleteOptionsBuilder().withApiVersion(seldonDeployment.getApiVersion()).build());
       } catch (ApiException e1) {
         LOG.error(e1.getMessage(), e1);
       }
@@ -570,16 +568,15 @@ public class K8sSubmitter implements Submitter {
               seldonDeployment.getVersion(),
               "default",
               seldonDeployment.getPlural(),
-              seldonDeployment.getMetadata().getName(),
-              new 
V1DeleteOptionsBuilder().withApiVersion(seldonDeployment.getApiVersion()).build(),
-              null, null, null);
+              seldonDeployment.getMetadata().getName(), null, null, null,
+              null, new 
V1DeleteOptionsBuilder().withApiVersion(seldonDeployment.getApiVersion()).build());
       api.deleteNamespacedCustomObject(istioVirtualService.getGroup(),
               istioVirtualService.getVersion(),
               "default",
               istioVirtualService.getPlural(),
               istioVirtualService.getMetadata().getName(),
-              new 
V1DeleteOptionsBuilder().withApiVersion(istioVirtualService.getApiVersion()).build(),
-              null, null, null);
+              null, null, null, null,
+              new 
V1DeleteOptionsBuilder().withApiVersion(istioVirtualService.getApiVersion()).build());
     } catch (ApiException e) {
       LOG.error(e.getMessage(), e);
       throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
@@ -602,8 +599,9 @@ public class K8sSubmitter implements Submitter {
             null,
             null,
             null,
-            Boolean.TRUE,
             null,
+            null,
+            Boolean.TRUE,
             null
         ),
         new TypeToken<Watch.Response<MLJob>>() {
@@ -644,8 +642,9 @@ public class K8sSubmitter implements Submitter {
             null,
             null,
             null,
-            Boolean.TRUE,
             null,
+            null,
+            Boolean.TRUE,
             null
         ),
         new TypeToken<Watch.Response<MLJob>>() {
@@ -697,7 +696,7 @@ public class K8sSubmitter implements Submitter {
     but it can still work fine and delete the PVC
     */
     try {
-      V1Status result = coreApi.deleteNamespacedPersistentVolumeClaim(
+      V1PersistentVolumeClaim result = 
coreApi.deleteNamespacedPersistentVolumeClaim(
           pvcName, namespace, "true",
           null, null, null,
           null, null
@@ -741,7 +740,7 @@ public class K8sSubmitter implements Submitter {
       api.createNamespacedCustomObject(
           ingressRoute.getGroup(), ingressRoute.getVersion(),
           ingressRoute.getMetadata().getNamespace(),
-          ingressRoute.getPlural(), ingressRoute, "true");
+          ingressRoute.getPlural(), ingressRoute, "true", null, null);
     } catch (ApiException e) {
       LOG.error("K8s submitter: Create Traefik custom resource object failed 
by " + e.getMessage(), e);
       throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
@@ -755,9 +754,8 @@ public class K8sSubmitter implements Submitter {
     try {
       api.deleteNamespacedCustomObject(
           IngressRoute.CRD_INGRESSROUTE_GROUP_V1, 
IngressRoute.CRD_INGRESSROUTE_VERSION_V1,
-          namespace, IngressRoute.CRD_INGRESSROUTE_PLURAL_V1, name,
-          new 
V1DeleteOptionsBuilder().withApiVersion(IngressRoute.CRD_APIVERSION_V1).build(),
-          null, null, null);
+          namespace, IngressRoute.CRD_INGRESSROUTE_PLURAL_V1, name, null, 
null, null,
+              null, new 
V1DeleteOptionsBuilder().withApiVersion(IngressRoute.CRD_APIVERSION_V1).build());
     } catch (ApiException e) {
       LOG.error("K8s submitter: Delete Traefik custom resource object failed 
by " + e.getMessage(), e);
       throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
@@ -815,9 +813,8 @@ public class K8sSubmitter implements Submitter {
     try {
       Object object = api.deleteNamespacedCustomObject(notebookCR.getGroup(), 
notebookCR.getVersion(),
           namespace, notebookCR.getPlural(),
-          notebookCR.getMetadata().getName(),
-          new 
V1DeleteOptionsBuilder().withApiVersion(notebookCR.getApiVersion()).build(),
-          null, null, null);
+          notebookCR.getMetadata().getName(), null, null, null, null,
+          new 
V1DeleteOptionsBuilder().withApiVersion(notebookCR.getApiVersion()).build());
     } catch (ApiException e) {
       throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
     }
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/AbstractCodeLocalizer.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/AbstractCodeLocalizer.java
index 46b359b..ff25f27 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/AbstractCodeLocalizer.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/AbstractCodeLocalizer.java
@@ -21,9 +21,9 @@ package 
org.apache.submarine.server.submitter.k8s.experiment.codelocalizer;
 
 import org.apache.submarine.server.api.exception.InvalidSpecException;
 
-import io.kubernetes.client.models.V1EmptyDirVolumeSource;
-import io.kubernetes.client.models.V1PodSpec;
-import io.kubernetes.client.models.V1Volume;
+import io.kubernetes.client.openapi.models.V1EmptyDirVolumeSource;
+import io.kubernetes.client.openapi.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1Volume;
 
 public abstract class AbstractCodeLocalizer implements CodeLocalizer {
 
@@ -32,18 +32,18 @@ public abstract class AbstractCodeLocalizer implements 
CodeLocalizer {
   public static final String CODE_LOCALIZER_INIT_CONTAINER_NAME = 
"code-localizer";
   public static final String CODE_LOCALIZER_PATH_ENV_VAR = "CODE_PATH";
   private String url;
-  
+
   public AbstractCodeLocalizer(String url) {
     this.url = url;
   }
-  
+
   /**
    * @return the url
    */
   public String getUrl() {
     return url;
   }
-  
+
   @Override
   public void localize(V1PodSpec podSpec) {
     V1Volume volume = new V1Volume();
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/CodeLocalizer.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/CodeLocalizer.java
index 2e3e7c1..7c7ca86 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/CodeLocalizer.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/CodeLocalizer.java
@@ -19,7 +19,7 @@
 
 package org.apache.submarine.server.submitter.k8s.experiment.codelocalizer;
 
-import io.kubernetes.client.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1PodSpec;
 
 public interface CodeLocalizer {
 
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/DummyCodeLocalizer.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/DummyCodeLocalizer.java
index daf682f..4f08f27 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/DummyCodeLocalizer.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/DummyCodeLocalizer.java
@@ -19,7 +19,7 @@
 
 package org.apache.submarine.server.submitter.k8s.experiment.codelocalizer;
 
-import io.kubernetes.client.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1PodSpec;
 
 public class DummyCodeLocalizer extends AbstractCodeLocalizer {
 
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/GitCodeLocalizer.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/GitCodeLocalizer.java
index f507e0b..bded2c3 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/GitCodeLocalizer.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/GitCodeLocalizer.java
@@ -26,10 +26,10 @@ import java.util.List;
 
 import org.apache.submarine.server.api.exception.InvalidSpecException;
 
-import io.kubernetes.client.models.V1Container;
-import io.kubernetes.client.models.V1EnvVar;
-import io.kubernetes.client.models.V1PodSpec;
-import io.kubernetes.client.models.V1VolumeMount;
+import io.kubernetes.client.openapi.models.V1Container;
+import io.kubernetes.client.openapi.models.V1EnvVar;
+import io.kubernetes.client.openapi.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1VolumeMount;
 
 public abstract class GitCodeLocalizer extends AbstractCodeLocalizer {
 
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/HTTPGitCodeLocalizer.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/HTTPGitCodeLocalizer.java
index edb475b..472135f 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/HTTPGitCodeLocalizer.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/HTTPGitCodeLocalizer.java
@@ -19,14 +19,14 @@
 
 package org.apache.submarine.server.submitter.k8s.experiment.codelocalizer;
 
-import io.kubernetes.client.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1PodSpec;
 
 public class HTTPGitCodeLocalizer extends GitCodeLocalizer {
 
   public HTTPGitCodeLocalizer(String url) {
     super(url);
   }
-  
+
   @Override
   public void localize(V1PodSpec podSpec) {
     super.localize(podSpec);
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/SSHGitCodeLocalizer.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/SSHGitCodeLocalizer.java
index 085ae0b..d49ef3e 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/SSHGitCodeLocalizer.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/experiment/codelocalizer/SSHGitCodeLocalizer.java
@@ -21,11 +21,11 @@ package 
org.apache.submarine.server.submitter.k8s.experiment.codelocalizer;
 
 import java.util.List;
 
-import io.kubernetes.client.models.V1Container;
-import io.kubernetes.client.models.V1EnvVar;
-import io.kubernetes.client.models.V1PodSpec;
-import io.kubernetes.client.models.V1SecurityContext;
-import io.kubernetes.client.models.V1VolumeMount;
+import io.kubernetes.client.openapi.models.V1Container;
+import io.kubernetes.client.openapi.models.V1EnvVar;
+import io.kubernetes.client.openapi.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1SecurityContext;
+import io.kubernetes.client.openapi.models.V1VolumeMount;
 
 public class SSHGitCodeLocalizer extends GitCodeLocalizer {
 
@@ -36,7 +36,7 @@ public class SSHGitCodeLocalizer extends GitCodeLocalizer {
   public static final long GIT_SYNC_USER = 65533L;
   public static final String GIT_SYNC_SSH_NAME = "GIT_SYNC_SSH";
   public static final String GIT_SYNC_SSH_VALUE = "true";
-  
+
   public SSHGitCodeLocalizer(String url) {
     super(url);
   }
@@ -51,14 +51,14 @@ public class SSHGitCodeLocalizer extends GitCodeLocalizer {
         sshEnv.setName(GIT_SYNC_SSH_NAME);
         sshEnv.setValue(GIT_SYNC_SSH_VALUE);
         gitSyncEnvVars.add(sshEnv);
-        
+
         List<V1VolumeMount> mounts = container.getVolumeMounts();
         V1VolumeMount mount = new V1VolumeMount();
         mount.setName(GIT_SECRET_MOUNT_NAME);
         mount.setMountPath(GIT_SECRET_PATH);
         mount.setReadOnly(true);
         mounts.add(mount);
-        
+
         V1SecurityContext containerSecurityContext =
             new V1SecurityContext();
         containerSecurityContext
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ListMeta.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ListMeta.java
index 7ed10f5..13b16f4 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ListMeta.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ListMeta.java
@@ -20,7 +20,7 @@
 package org.apache.submarine.server.submitter.k8s.model;
 
 import com.google.gson.GsonBuilder;
-import io.kubernetes.client.models.V1ListMeta;
+import io.kubernetes.client.openapi.models.V1ListMeta;
 
 /**
  * ListMeta describes metadata that synthetic resources must have, including 
lists and various
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/MLJob.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/MLJob.java
index af77365..7673836 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/MLJob.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/MLJob.java
@@ -20,8 +20,8 @@
 package org.apache.submarine.server.submitter.k8s.model;
 
 import com.google.gson.annotations.SerializedName;
-import io.kubernetes.client.models.V1JobStatus;
-import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1JobStatus;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
 
 /**
  * The machine learning job for the CRD job.
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/MLJobReplicaSpec.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/MLJobReplicaSpec.java
index 83919e3..ad6947e 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/MLJobReplicaSpec.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/MLJobReplicaSpec.java
@@ -20,7 +20,7 @@
 package org.apache.submarine.server.submitter.k8s.model;
 
 import com.google.gson.annotations.SerializedName;
-import io.kubernetes.client.models.V1PodTemplateSpec;
+import io.kubernetes.client.openapi.models.V1PodTemplateSpec;
 
 import java.math.BigDecimal;
 
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
index c77a1f2..143d4a4 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCR.java
@@ -20,7 +20,7 @@
 package org.apache.submarine.server.submitter.k8s.model;
 
 import com.google.gson.annotations.SerializedName;
-import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
 
 public class NotebookCR {
 
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCRSpec.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCRSpec.java
index bd5b115..9751720 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCRSpec.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookCRSpec.java
@@ -20,8 +20,8 @@
 package org.apache.submarine.server.submitter.k8s.model;
 
 import com.google.gson.annotations.SerializedName;
-import io.kubernetes.client.models.V1EnvVar;
-import io.kubernetes.client.models.V1PodTemplateSpec;
+import io.kubernetes.client.openapi.models.V1EnvVar;
+import io.kubernetes.client.openapi.models.V1PodTemplateSpec;
 
 import java.math.BigDecimal;
 import java.util.List;
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookStatus.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookStatus.java
index bc11006..9e50523 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookStatus.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/NotebookStatus.java
@@ -19,7 +19,7 @@
 package org.apache.submarine.server.submitter.k8s.model;
 
 import com.google.gson.annotations.SerializedName;
-import io.kubernetes.client.models.V1ContainerState;
+import io.kubernetes.client.openapi.models.V1ContainerState;
 
 import java.util.List;
 
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRoute.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRoute.java
index 72c5fc6..35e0aa6 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRoute.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRoute.java
@@ -20,7 +20,7 @@
 package org.apache.submarine.server.submitter.k8s.model.ingressroute;
 
 import com.google.gson.annotations.SerializedName;
-import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
 
 public class IngressRoute {
   public static final String CRD_INGRESSROUTE_GROUP_V1 = "traefik.containo.us";
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/middlewares/Middlewares.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/middlewares/Middlewares.java
index 7b3f731..6547fcb 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/middlewares/Middlewares.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/middlewares/Middlewares.java
@@ -20,7 +20,7 @@
 package org.apache.submarine.server.submitter.k8s.model.middlewares;
 
 import com.google.gson.annotations.SerializedName;
-import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
 
 public class Middlewares {
   // reference: 
https://doc.traefik.io/traefik/reference/dynamic-configuration/kubernetes-crd/#definitions
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
index ece4be7..a5431fb 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
@@ -21,17 +21,17 @@ package org.apache.submarine.server.submitter.k8s.parser;
 
 import io.kubernetes.client.custom.Quantity;
 
-import io.kubernetes.client.models.V1Container;
-import io.kubernetes.client.models.V1EnvVar;
-import io.kubernetes.client.models.V1ObjectMeta;
-import io.kubernetes.client.models.V1PersistentVolumeClaimVolumeSource;
-import io.kubernetes.client.models.V1PodSecurityContext;
-import io.kubernetes.client.models.V1PodSpec;
-import io.kubernetes.client.models.V1PodTemplateSpec;
-import io.kubernetes.client.models.V1ResourceRequirements;
-import io.kubernetes.client.models.V1SecretVolumeSource;
-import io.kubernetes.client.models.V1Volume;
-import io.kubernetes.client.models.V1VolumeMount;
+import io.kubernetes.client.openapi.models.V1Container;
+import io.kubernetes.client.openapi.models.V1EnvVar;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimVolumeSource;
+import io.kubernetes.client.openapi.models.V1PodSecurityContext;
+import io.kubernetes.client.openapi.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1PodTemplateSpec;
+import io.kubernetes.client.openapi.models.V1ResourceRequirements;
+import io.kubernetes.client.openapi.models.V1SecretVolumeSource;
+import io.kubernetes.client.openapi.models.V1Volume;
+import io.kubernetes.client.openapi.models.V1VolumeMount;
 import org.apache.submarine.commons.utils.SubmarineConfVars;
 import org.apache.submarine.commons.utils.SubmarineConfiguration;
 import org.apache.submarine.server.api.environment.Environment;
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
index b6ac276..0ed30b1 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
@@ -20,15 +20,15 @@
 package org.apache.submarine.server.submitter.k8s.parser;
 
 import io.kubernetes.client.custom.Quantity;
-import io.kubernetes.client.models.V1Container;
-import io.kubernetes.client.models.V1EnvVar;
-import io.kubernetes.client.models.V1ObjectMeta;
-import io.kubernetes.client.models.V1PodTemplateSpec;
-import io.kubernetes.client.models.V1PodSpec;
-import io.kubernetes.client.models.V1ResourceRequirements;
-import io.kubernetes.client.models.V1Volume;
-import io.kubernetes.client.models.V1VolumeMount;
-import io.kubernetes.client.models.V1PersistentVolumeClaimVolumeSource;
+import io.kubernetes.client.openapi.models.V1Container;
+import io.kubernetes.client.openapi.models.V1EnvVar;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1PodTemplateSpec;
+import io.kubernetes.client.openapi.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1ResourceRequirements;
+import io.kubernetes.client.openapi.models.V1Volume;
+import io.kubernetes.client.openapi.models.V1VolumeMount;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimVolumeSource;
 
 import org.apache.submarine.commons.utils.SubmarineConfVars;
 import org.apache.submarine.commons.utils.SubmarineConfiguration;
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ServeSpecParser.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ServeSpecParser.java
index a901419..0e42568 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ServeSpecParser.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ServeSpecParser.java
@@ -27,19 +27,19 @@ import 
org.apache.submarine.server.submitter.k8s.model.middlewares.MiddlewaresSp
 import org.apache.submarine.server.submitter.k8s.model.middlewares.StripPrefix;
 
 import io.kubernetes.client.custom.IntOrString;
-import io.kubernetes.client.models.V1Container;
-import io.kubernetes.client.models.V1ContainerPort;
-import io.kubernetes.client.models.V1Deployment;
-import io.kubernetes.client.models.V1DeploymentSpec;
-import io.kubernetes.client.models.V1HTTPGetAction;
-import io.kubernetes.client.models.V1LabelSelector;
-import io.kubernetes.client.models.V1ObjectMeta;
-import io.kubernetes.client.models.V1PodSpec;
-import io.kubernetes.client.models.V1PodTemplateSpec;
-import io.kubernetes.client.models.V1Probe;
-import io.kubernetes.client.models.V1Service;
-import io.kubernetes.client.models.V1ServicePort;
-import io.kubernetes.client.models.V1ServiceSpec;
+import io.kubernetes.client.openapi.models.V1Container;
+import io.kubernetes.client.openapi.models.V1ContainerPort;
+import io.kubernetes.client.openapi.models.V1Deployment;
+import io.kubernetes.client.openapi.models.V1DeploymentSpec;
+import io.kubernetes.client.openapi.models.V1HTTPGetAction;
+import io.kubernetes.client.openapi.models.V1LabelSelector;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1PodTemplateSpec;
+import io.kubernetes.client.openapi.models.V1Probe;
+import io.kubernetes.client.openapi.models.V1Service;
+import io.kubernetes.client.openapi.models.V1ServicePort;
+import io.kubernetes.client.openapi.models.V1ServiceSpec;
 
 import java.util.ArrayList;
 import java.util.Arrays;
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParser.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParser.java
index cde0cce..ebd080f 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParser.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParser.java
@@ -20,13 +20,13 @@
 package org.apache.submarine.server.submitter.k8s.parser;
 
 import io.kubernetes.client.custom.Quantity;
-import io.kubernetes.client.models.V1HostPathVolumeSource;
-import io.kubernetes.client.models.V1ObjectMeta;
-import io.kubernetes.client.models.V1PersistentVolume;
-import io.kubernetes.client.models.V1PersistentVolumeClaim;
-import io.kubernetes.client.models.V1PersistentVolumeClaimSpec;
-import io.kubernetes.client.models.V1PersistentVolumeSpec;
-import io.kubernetes.client.models.V1ResourceRequirements;
+import io.kubernetes.client.openapi.models.V1HostPathVolumeSource;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1PersistentVolume;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimSpec;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeSpec;
+import io.kubernetes.client.openapi.models.V1ResourceRequirements;
 
 import java.util.Collections;
 
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/MLJobConverter.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/MLJobConverter.java
index 93f9deb..f6e4e8c 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/MLJobConverter.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/MLJobConverter.java
@@ -20,12 +20,12 @@
 package org.apache.submarine.server.submitter.k8s.util;
 
 import java.util.List;
-import io.kubernetes.client.models.V1DeleteOptions;
-import io.kubernetes.client.models.V1DeleteOptionsBuilder;
-import io.kubernetes.client.models.V1JobCondition;
-import io.kubernetes.client.models.V1JobStatus;
-import io.kubernetes.client.models.V1Status;
-import io.kubernetes.client.models.V1StatusDetails;
+import io.kubernetes.client.openapi.models.V1DeleteOptions;
+import io.kubernetes.client.openapi.models.V1DeleteOptionsBuilder;
+import io.kubernetes.client.openapi.models.V1JobCondition;
+import io.kubernetes.client.openapi.models.V1JobStatus;
+import io.kubernetes.client.openapi.models.V1Status;
+import io.kubernetes.client.openapi.models.V1StatusDetails;
 import org.apache.submarine.server.api.experiment.Experiment;
 import org.apache.submarine.server.submitter.k8s.model.MLJob;
 import org.joda.time.DateTime;
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
index 60e57ab..c567520 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/NotebookUtils.java
@@ -21,9 +21,9 @@ package org.apache.submarine.server.submitter.k8s.util;
 
 import com.google.gson.Gson;
 import com.google.gson.JsonSyntaxException;
-import io.kubernetes.client.JSON;
-import io.kubernetes.client.models.V1ContainerState;
-import io.kubernetes.client.models.V1Status;
+import io.kubernetes.client.openapi.JSON;
+import io.kubernetes.client.openapi.models.V1ContainerState;
+import io.kubernetes.client.openapi.models.V1Status;
 import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
 import org.apache.submarine.server.api.notebook.Notebook;
 import org.apache.submarine.server.submitter.k8s.model.NotebookCR;
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/OwnerReferenceUtils.java
 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/OwnerReferenceUtils.java
index 4a319d1..3951ac3 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/OwnerReferenceUtils.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/OwnerReferenceUtils.java
@@ -21,7 +21,7 @@ package org.apache.submarine.server.submitter.k8s.util;
 
 import java.util.ArrayList;
 
-import io.kubernetes.client.models.V1OwnerReference;
+import io.kubernetes.client.openapi.models.V1OwnerReference;
 
 public class OwnerReferenceUtils {
   private static final String SUBMARINE_APIVERSION = "SUBMARINE_APIVERSION";
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
 
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
index e9df5f9..b2c732e 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
@@ -24,8 +24,8 @@ import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.List;
 
-import io.kubernetes.client.models.V1ObjectMeta;
-import io.kubernetes.client.models.V1Volume;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1Volume;
 
 import org.apache.submarine.commons.utils.SubmarineConfVars;
 import org.apache.submarine.commons.utils.SubmarineConfiguration;
@@ -49,9 +49,9 @@ import 
org.apache.submarine.server.submitter.k8s.experiment.codelocalizer.GitCod
 import 
org.apache.submarine.server.submitter.k8s.experiment.codelocalizer.SSHGitCodeLocalizer;
 import org.junit.Assert;
 import org.junit.Test;
-import io.kubernetes.client.models.V1Container;
-import io.kubernetes.client.models.V1EmptyDirVolumeSource;
-import io.kubernetes.client.models.V1EnvVar;
+import io.kubernetes.client.openapi.models.V1Container;
+import io.kubernetes.client.openapi.models.V1EmptyDirVolumeSource;
+import io.kubernetes.client.openapi.models.V1EnvVar;
 
 
 public class ExperimentSpecParserTest extends SpecBuilder {
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
 
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
index 64678fc..8e25cc6 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
@@ -22,7 +22,7 @@ package org.apache.submarine.server.submitter.k8s;
 import java.io.IOException;
 import java.net.URISyntaxException;
 
-import io.kubernetes.client.ApiException;
+import io.kubernetes.client.openapi.ApiException;
 import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
 import org.apache.submarine.server.api.experiment.Experiment;
 import org.apache.submarine.server.api.experiment.TensorboardInfo;
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/MLJobConverterTest.java
 
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/MLJobConverterTest.java
index 851f658..4e9942e 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/MLJobConverterTest.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/MLJobConverterTest.java
@@ -24,13 +24,13 @@ import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.List;
 
-import io.kubernetes.client.models.V1DeleteOptions;
-import io.kubernetes.client.models.V1JobCondition;
-import io.kubernetes.client.models.V1JobConditionBuilder;
-import io.kubernetes.client.models.V1JobStatus;
-import io.kubernetes.client.models.V1JobStatusBuilder;
-import io.kubernetes.client.models.V1Status;
-import io.kubernetes.client.models.V1StatusBuilder;
+import io.kubernetes.client.openapi.models.V1DeleteOptions;
+import io.kubernetes.client.openapi.models.V1JobCondition;
+import io.kubernetes.client.openapi.models.V1JobConditionBuilder;
+import io.kubernetes.client.openapi.models.V1JobStatus;
+import io.kubernetes.client.openapi.models.V1JobStatusBuilder;
+import io.kubernetes.client.openapi.models.V1Status;
+import io.kubernetes.client.openapi.models.V1StatusBuilder;
 import org.apache.submarine.server.api.exception.InvalidSpecException;
 import org.apache.submarine.server.api.experiment.Experiment;
 import org.apache.submarine.server.api.spec.ExperimentSpec;
diff --git 
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
 
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
index a0d680b..471301a 100644
--- 
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
+++ 
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/NotebookSpecParserTest.java
@@ -19,8 +19,8 @@
 
 package org.apache.submarine.server.submitter.k8s;
 
-import io.kubernetes.client.models.V1EnvVar;
-import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1EnvVar;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import org.apache.submarine.server.api.spec.NotebookMeta;
 import org.apache.submarine.server.api.spec.NotebookPodSpec;
 import org.apache.submarine.server.api.spec.NotebookSpec;
diff --git 
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
 
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
index 1cbd386..e9aa230 100644
--- 
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
+++ 
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
@@ -57,11 +57,11 @@ import com.google.gson.GsonBuilder;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
 
-import io.kubernetes.client.ApiClient;
-import io.kubernetes.client.ApiException;
-import io.kubernetes.client.Configuration;
-import io.kubernetes.client.JSON;
-import io.kubernetes.client.apis.CustomObjectsApi;
+import io.kubernetes.client.openapi.ApiClient;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.Configuration;
+import io.kubernetes.client.openapi.JSON;
+import io.kubernetes.client.openapi.apis.CustomObjectsApi;
 import io.kubernetes.client.util.ClientBuilder;
 import io.kubernetes.client.util.KubeConfig;
 
diff --git 
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/NotebookRestApiIT.java
 
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/NotebookRestApiIT.java
index ada557f..2f88205 100644
--- 
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/NotebookRestApiIT.java
+++ 
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/NotebookRestApiIT.java
@@ -24,11 +24,11 @@ import com.google.gson.GsonBuilder;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
-import io.kubernetes.client.ApiClient;
-import io.kubernetes.client.ApiException;
-import io.kubernetes.client.Configuration;
-import io.kubernetes.client.JSON;
-import io.kubernetes.client.apis.CustomObjectsApi;
+import io.kubernetes.client.openapi.ApiClient;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.Configuration;
+import io.kubernetes.client.openapi.JSON;
+import io.kubernetes.client.openapi.apis.CustomObjectsApi;
 import io.kubernetes.client.util.ClientBuilder;
 import io.kubernetes.client.util.KubeConfig;
 import org.apache.commons.httpclient.methods.DeleteMethod;

---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to