This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 4ef6c66 SUBMARINE-1082. Create PyTorch serving
4ef6c66 is described below
commit 4ef6c66b8c388af2c83cc3536e4c9247f9825b9a
Author: KUAN-HSUN-LI <[email protected]>
AuthorDate: Sat Nov 27 12:33:58 2021 +0800
SUBMARINE-1082. Create PyTorch serving
### What is this PR for?
Provide PyTorch model serving using `triton inference server` in
`seldon-core`
### What type of PR is it?
[Feature]
### Todos
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-1082
### How should this be tested?
I will provide testing cases in another PR.
### Screenshots (if appropriate)
https://user-images.githubusercontent.com/38066413/143668211-e2e06324-de40-4183-bbf8-a09c6b484d29.mp4
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? Yes
Author: KUAN-HSUN-LI <[email protected]>
Signed-off-by: Kevin <[email protected]>
Closes #811 from KUAN-HSUN-LI/SUBMARINE-1082 and squashes the following
commits:
68c8f20c [KUAN-HSUN-LI] SUBMARINE-1082. Create PyTorch serving
---
submarine-serve/installation/install.sh | 2 +-
.../serve/pytorch/SeldonPytorchServing.java | 45 ++++++++++++++++++++++
.../submarine/serve/utils/SeldonConstants.java | 5 +++
.../org/apache/submarine/server/api/Submitter.java | 4 +-
.../submarine/server/model/ModelManager.java | 37 +++++++++++-------
.../server/submitter/k8s/K8sSubmitter.java | 17 ++++----
website/docs/api/serve.md | 6 +--
7 files changed, 86 insertions(+), 30 deletions(-)
diff --git a/submarine-serve/installation/install.sh
b/submarine-serve/installation/install.sh
index 6eab364..75734a8 100755
--- a/submarine-serve/installation/install.sh
+++ b/submarine-serve/installation/install.sh
@@ -41,5 +41,5 @@ helm install seldon-core seldon-core-operator \
--namespace seldon-system \
--version 1.10.0 \
--set istio.enabled=true \
- --set executor.defaultEnvSecretRefName=seldon-core-init-container-secret
2> /dev/null
+ --set executor.defaultEnvSecretRefName=submarine-serve-secret 2> /dev/null
kubectl apply -f ${CURRENT_PATH}/seldon-gateway.yaml
diff --git
a/submarine-serve/src/main/java/org/apache/submarine/serve/pytorch/SeldonPytorchServing.java
b/submarine-serve/src/main/java/org/apache/submarine/serve/pytorch/SeldonPytorchServing.java
new file mode 100644
index 0000000..b7881d6
--- /dev/null
+++
b/submarine-serve/src/main/java/org/apache/submarine/serve/pytorch/SeldonPytorchServing.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.submarine.serve.pytorch;
+
+import io.kubernetes.client.models.V1ObjectMeta;
+import org.apache.submarine.serve.seldon.SeldonDeployment;
+import org.apache.submarine.serve.seldon.SeldonGraph;
+import org.apache.submarine.serve.seldon.SeldonPredictor;
+import org.apache.submarine.serve.utils.SeldonConstants;
+
+public class SeldonPytorchServing extends SeldonDeployment {
+ public SeldonPytorchServing(String name, String modelURI){
+ V1ObjectMeta v1ObjectMeta = new V1ObjectMeta();
+ v1ObjectMeta.setName(name);
+ v1ObjectMeta.setNamespace(SeldonConstants.DEFAULT_NAMESPACE);
+ setMetadata(v1ObjectMeta);
+
+ setSpec(new SeldonDeploymentSpec(SeldonConstants.KFSERVING_PROTOCOL));
+
+ SeldonGraph seldonGraph = new SeldonGraph();
+ seldonGraph.setName(name);
+ seldonGraph.setImplementation(SeldonConstants.TRITON_IMPLEMENTATION);
+ seldonGraph.setModelUri(modelURI);
+ SeldonPredictor seldonPredictor = new SeldonPredictor();
+ seldonPredictor.setSeldonGraph(seldonGraph);
+
+ addPredictor(seldonPredictor);
+ }
+}
diff --git
a/submarine-serve/src/main/java/org/apache/submarine/serve/utils/SeldonConstants.java
b/submarine-serve/src/main/java/org/apache/submarine/serve/utils/SeldonConstants.java
index bcaf0a6..e41a564 100644
---
a/submarine-serve/src/main/java/org/apache/submarine/serve/utils/SeldonConstants.java
+++
b/submarine-serve/src/main/java/org/apache/submarine/serve/utils/SeldonConstants.java
@@ -37,6 +37,11 @@ public class SeldonConstants {
public static final String SELDON_PROTOCOL = "seldon";
+ public static final String KFSERVING_PROTOCOL = "kfserving";
+
// TensorFlow
public static final String TFSERVING_IMPLEMENTATION = "TENSORFLOW_SERVER";
+
+ // PyTorch
+ public static final String TRITON_IMPLEMENTATION = "TRITON_SERVER";
}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
index 96e0f2b..b1066a5 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
@@ -25,7 +25,6 @@ import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.ExperimentLog;
import org.apache.submarine.server.api.experiment.TensorboardInfo;
import org.apache.submarine.server.api.experiment.MlflowInfo;
-import org.apache.submarine.server.api.model.ServeResponse;
import org.apache.submarine.server.api.model.ServeSpec;
import org.apache.submarine.server.api.notebook.Notebook;
import org.apache.submarine.server.api.spec.ExperimentSpec;
@@ -127,10 +126,9 @@ public interface Submitter {
/**
* Create Serve with spec
* @param spec
- * @return object
* @throws SubmarineRuntimeException running error
*/
- ServeResponse createServe(ServeSpec spec) throws SubmarineRuntimeException;
+ void createServe(ServeSpec spec) throws SubmarineRuntimeException;
/**
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/model/ModelManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/model/ModelManager.java
index 49ceb6e..3a6aa65 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/model/ModelManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/model/ModelManager.java
@@ -19,7 +19,6 @@
package org.apache.submarine.server.model;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.core.Response;
@@ -29,6 +28,7 @@ import org.apache.submarine.server.SubmitterManager;
import org.apache.submarine.server.api.Submitter;
import org.apache.submarine.server.api.model.ServeResponse;
import org.apache.submarine.server.api.model.ServeSpec;
+import org.apache.submarine.server.model.database.entities.ModelVersionEntity;
import org.apache.submarine.server.model.database.service.ModelVersionService;
@@ -67,11 +67,14 @@ public class ModelManager {
* Create a model serve.
*/
public ServeResponse createServe(ServeSpec spec) throws
SubmarineRuntimeException {
- checkServeSpec(spec);
- String modelURI = modelVersionService.select(spec.getModelName(),
spec.getModelVersion()).getSource();
- spec.setModelURI(modelURI);
+ setServeInfo(spec);
+
+
+ LOG.info("Create " + spec.getModelType() + " model serve");
- ServeResponse serveResponse = submitter.createServe(spec);
+ submitter.createServe(spec);
+
+ ServeResponse serveResponse = getServeResponse(spec);
return serveResponse;
}
@@ -79,9 +82,9 @@ public class ModelManager {
* Delete a model serve.
*/
public void deleteServe(ServeSpec spec) throws SubmarineRuntimeException {
- checkServeSpec(spec);
- String modelURI = modelVersionService.select(spec.getModelName(),
spec.getModelVersion()).getSource();
- spec.setModelURI(modelURI);
+ setServeInfo(spec);
+
+ LOG.info("Delete " + spec.getModelType() + " model serve");
submitter.deleteServe(spec);
}
@@ -100,11 +103,19 @@ public class ModelManager {
throw new SubmarineRuntimeException(Response.Status.OK.getStatusCode(),
"Invalid. Model version must be positive, but get " +
modelVersion);
}
- String modelType = spec.getModelType();
- if (!modelType.equals("tensorflow") && !modelType.equals("pytorch")) {
- throw new SubmarineRuntimeException(Response.Status.OK.getStatusCode(),
- "Invalid. Serve Type can only be tensorflow or pytorch, but
get " + modelType);
- }
}
}
+
+ private void setServeInfo(ServeSpec spec){
+ checkServeSpec(spec);
+
+ // Get model type and model uri from DB and set the value in the spec.
+ ModelVersionEntity modelVersion =
modelVersionService.select(spec.getModelName(), spec.getModelVersion());
+ spec.setModelURI(modelVersion.getSource());
+ spec.setModelType(modelVersion.getModelType());
+ }
+
+ private ServeResponse getServeResponse(ServeSpec spec){
+ return new ServeResponse();
+ }
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
index f99d69f..c1e8298 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
@@ -23,12 +23,13 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
@@ -50,21 +51,21 @@ import io.kubernetes.client.models.V1PersistentVolumeClaim;
import io.kubernetes.client.models.V1Pod;
import io.kubernetes.client.models.V1PodList;
import io.kubernetes.client.models.V1Status;
-import io.kubernetes.client.util.Watch;
import io.kubernetes.client.util.ClientBuilder;
import io.kubernetes.client.util.KubeConfig;
+import io.kubernetes.client.util.Watch;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
+import org.apache.submarine.serve.pytorch.SeldonPytorchServing;
import org.apache.submarine.serve.seldon.SeldonDeployment;
import org.apache.submarine.serve.tensorflow.SeldonTFServing;
import org.apache.submarine.server.api.Submitter;
import org.apache.submarine.server.api.exception.InvalidSpecException;
import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.ExperimentLog;
-import org.apache.submarine.server.api.experiment.TensorboardInfo;
import org.apache.submarine.server.api.experiment.MlflowInfo;
-import org.apache.submarine.server.api.model.ServeResponse;
+import org.apache.submarine.server.api.experiment.TensorboardInfo;
import org.apache.submarine.server.api.model.ServeSpec;
import org.apache.submarine.server.api.notebook.Notebook;
import org.apache.submarine.server.api.spec.ExperimentMeta;
@@ -518,7 +519,7 @@ public class K8sSubmitter implements Submitter {
}
@Override
- public ServeResponse createServe(ServeSpec spec)
+ public void createServe(ServeSpec spec)
throws SubmarineRuntimeException {
SeldonDeployment seldonDeployment = parseServeSpec(spec);
@@ -533,7 +534,6 @@ public class K8sSubmitter implements Submitter {
LOG.error(e.getMessage(), e);
throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
}
- return new ServeResponse();
}
@Override
@@ -763,8 +763,7 @@ public class K8sSubmitter implements Submitter {
if (modelType.equals("tensorflow")){
seldonDeployment = new SeldonTFServing(modelName, modelURI);
} else if (modelType.equals("pytorch")){
- // TODO(KUAN-HSUN LI): create pytorch serve
- throw new SubmarineRuntimeException("Given serve type: " + modelType + "
is not supported.");
+ seldonDeployment = new SeldonPytorchServing(modelName, modelURI);
} else {
throw new SubmarineRuntimeException("Given serve type: " + modelType + "
is not supported.");
}
diff --git a/website/docs/api/serve.md b/website/docs/api/serve.md
index b4b179f..f1c3f47 100644
--- a/website/docs/api/serve.md
+++ b/website/docs/api/serve.md
@@ -23,7 +23,7 @@ under the License.
> Note: The Serv API is in the alpha stage which is subjected to incompatible
> changes in future releases.
-## Create a TensorFlow model serve
+## Create a model serve
`POST /api/v1/serve`
**Example Request**
@@ -34,7 +34,6 @@ curl -X POST -H "Content-Type: application/json" -d '
{
"modelName": "simple",
"modelVersion":1,
- "modelType":"tensorflow"
}
' http://127.0.0.1:32080/api/v1/serve
```
@@ -59,8 +58,7 @@ curl -X POST -H "Content-Type: application/json" -d '
curl -X DELETE -H "Content-Type: application/json" -d '
{
"modelName": "simple",
- "modelVersion":1,
- "modelType":"tensorflow"
+ "modelVersion":1,
}
' http://127.0.0.1:32080/api/v1/serve
```
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
