This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 6291724 SUBMARINE-1054. Refactor submarine-operator
6291724 is described below
commit 6291724ae1fd4dc8b48eae5332bf123e86375110
Author: Kenchu123 <[email protected]>
AuthorDate: Fri Nov 5 19:41:51 2021 +0800
SUBMARINE-1054. Refactor submarine-operator
### What is this PR for?
<!-- A few sentences describing the overall goals of the pull request's
commits.
First time? Check out the contributing guide -
https://submarine.apache.org/contribution/contributions.html
-->
The submarine operator is responsible for creating the submarine. However,
it will create the submarine with settings written in code currently.
We want to make it read settings from yaml files, so that others can easily
change the settings.
### What type of PR is it?
Improvement
### Todos
* [x] - Add artifacts
* [x] - Add YAML file parser
### What is the Jira issue?
<!-- * Open an issue on Jira
https://issues.apache.org/jira/browse/SUBMARINE/
* Put link here, and add [SUBMARINE-*Jira number*] in PR title, eg.
`SUBMARINE-23. PR title`
-->
https://issues.apache.org/jira/browse/SUBMARINE-1054
### How should this be tested?
<!--
* First time? Setup Travis CI as described on
https://submarine.apache.org/contribution/contributions.html#continuous-integration
* Strongly recommended: add automated unit tests for any new or changed
behavior
* Outline any manual steps to test the PR here.
-->
Run operator e2e test: `go test ./test/e2e`
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: Kenchu123 <[email protected]>
Signed-off-by: Kevin <[email protected]>
Closes #788 from Kenchu123/SUBMARINE-1054 and squashes the following
commits:
3e64c24c [Kenchu123] SUBMARINE-1054. Add mlflow initContainer to check
database
8e6bebf5 [Kenchu123] SUBMARINE-1054. Add database readinessProbe and change
the comment of PathToOSFile
37eca8a7 [Kenchu123] SUBMARINE-1054 Add artifacts and read files from them
---
.../artifacts/submarine/submarine-database.yaml | 80 ++++++++++
.../artifacts/submarine/submarine-ingress.yaml | 15 ++
.../artifacts/submarine/submarine-minio.yaml | 97 ++++++++++++
.../artifacts/submarine/submarine-mlflow.yaml | 121 +++++++++++++++
.../artifacts/submarine/submarine-rbac.yaml | 86 +++++++++++
.../artifacts/submarine/submarine-server.yaml | 90 +++++++++++
.../artifacts/submarine/submarine-tensorboard.yaml | 96 ++++++++++++
submarine-cloud-v2/dev.Dockerfile | 2 +
submarine-cloud-v2/pkg/controller/controller.go | 8 +
submarine-cloud-v2/pkg/controller/parser.go | 169 ++++++++++++++++++++
.../pkg/controller/submarine_database.go | 137 ++++-------------
.../pkg/controller/submarine_ingress.go | 37 ++---
.../pkg/controller/submarine_minio.go | 165 ++++----------------
.../pkg/controller/submarine_mlflow.go | 170 ++++-----------------
.../pkg/controller/submarine_server.go | 159 +++++++------------
.../pkg/controller/submarine_server_rbac.go | 64 ++------
.../pkg/controller/submarine_tensorboard.go | 162 ++++----------------
17 files changed, 956 insertions(+), 702 deletions(-)
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml
new file mode 100644
index 0000000..b461dd5
--- /dev/null
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml
@@ -0,0 +1,80 @@
+---
+# Source: submarine/templates/submarine-database.yaml
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Source: submarine/templates/submarine-database.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: submarine-database-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ storageClassName: submarine-storageclass
+ resources:
+ requests:
+ storage: 1Gi
+---
+# Source: submarine/templates/submarine-database.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: "submarine-database"
+spec:
+ ports:
+ - name: "submarine-database"
+ port: 3306
+ targetPort: 3306
+ selector:
+ app: "submarine-database"
+---
+# Source: submarine/templates/submarine-database.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: "submarine-database"
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: "submarine-database"
+ template:
+ metadata:
+ labels:
+ app: "submarine-database"
+
+ spec:
+ containers:
+ - name: "submarine-database"
+ image: "apache/submarine:database-0.7.0-SNAPSHOT"
+ imagePullPolicy: "IfNotPresent"
+ ports:
+ - containerPort: 3306
+ env:
+ - name: MYSQL_ROOT_PASSWORD
+ value: "password"
+ volumeMounts:
+ - mountPath: /var/lib/mysql
+ name: volume
+ subPath: submarine-database
+ readinessProbe:
+ tcpSocket:
+ port: 3306
+ volumes:
+ - name: volume
+ persistentVolumeClaim:
+ claimName: submarine-database-pvc
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-ingress.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-ingress.yaml
new file mode 100644
index 0000000..0f6b583
--- /dev/null
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-ingress.yaml
@@ -0,0 +1,15 @@
+---
+# Source: submarine/templates/submarine-ingress.yaml
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ name: submarine-server-ingress
+
+spec:
+ rules:
+ - http:
+ paths:
+ - path: /
+ backend:
+ serviceName: submarine-server
+ servicePort: 8080
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml
new file mode 100644
index 0000000..a764106
--- /dev/null
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml
@@ -0,0 +1,97 @@
+---
+# Source: submarine/templates/submarine-minio.yaml
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Source: submarine/templates/submarine-minio.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: submarine-minio-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ storageClassName: "submarine-storageclass"
+ resources:
+ requests:
+ storage: "10Gi"
+---
+# Source: submarine/templates/submarine-minio.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: submarine-minio-service
+spec:
+ type: ClusterIP
+ selector:
+ app: submarine-minio
+ ports:
+ - protocol: TCP
+ port: 9000
+ targetPort: 9000
+---
+# Source: submarine/templates/submarine-minio.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: submarine-minio
+spec:
+ selector:
+ matchLabels:
+ app: submarine-minio
+ template:
+ metadata:
+ labels:
+ app: submarine-minio
+ spec:
+ containers:
+ - name: submarine-minio-container
+ image: minio/minio:RELEASE.2021-02-14T04-01-33Z
+ imagePullPolicy: IfNotPresent
+ args:
+ - server
+ - /data
+ env:
+ - name: MINIO_ACCESS_KEY
+ value: "submarine_minio"
+ - name: MINIO_SECRET_KEY
+ value: "submarine_minio"
+ ports:
+ - containerPort: 9000
+ volumeMounts:
+ - mountPath: "/data"
+ name: "volume"
+ subPath: "submarine-minio"
+ volumes:
+ - name: "volume"
+ persistentVolumeClaim:
+ claimName: "submarine-minio-pvc"
+---
+# Source: submarine/templates/submarine-minio.yaml
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+ name: submarine-minio-ingressroute
+spec:
+ entryPoints:
+ - web
+ routes:
+ - kind: Rule
+ match: "PathPrefix(`/minio`)"
+ services:
+ - kind: Service
+ name: submarine-minio-service
+ port: 9000
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml
new file mode 100644
index 0000000..007fdfb
--- /dev/null
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml
@@ -0,0 +1,121 @@
+---
+# Source: submarine/templates/submarine-mlflow.yaml
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Source: submarine/templates/submarine-mlflow.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: submarine-mlflow-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ storageClassName: "submarine-storageclass"
+ resources:
+ requests:
+ storage: "10Gi"
+---
+# Source: submarine/templates/submarine-mlflow.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: submarine-mlflow-service
+spec:
+ type: ClusterIP
+ selector:
+ app: submarine-mlflow
+ ports:
+ - protocol: TCP
+ port: 5000
+ targetPort: 5000
+---
+# Source: submarine/templates/submarine-mlflow.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: submarine-mlflow
+spec:
+ selector:
+ matchLabels:
+ app: submarine-mlflow
+ template:
+ metadata:
+ labels:
+ app: submarine-mlflow
+ spec:
+ initContainers:
+ - name: check-database-connection
+ image: busybox:1.28
+ command: ["sh", "-c",
+ "until nc -z submarine-database 3306;
+ do echo waiting for database connection;
+ sleep 20; done"]
+ - name: submarine-mlflow-initcontainer
+ image: "minio/mc"
+ command: ["/bin/bash", "-c",
+ "cnt=0;
+ while ! /bin/bash -c 'mc config host add minio
http://submarine-minio-service:9000
+ submarine_minio submarine_minio' 2>&1;
+ do
+ sleep 15;
+ ((cnt=cnt+1));
+ if [ $cnt -eq 80 ];then
+ echo 'ERROR: wait too long for minio pod';
+ exit 1;
+ fi;
+ done;
+ if /bin/bash -c 'mc ls minio/mlflow' >/dev/null 2>&1; then
+ echo 'Bucket minio/mlflow already exists, skipping creation.';
+ else
+ /bin/bash -c 'mc mb minio/mlflow';
+ fi;"]
+ containers:
+ - name: submarine-mlflow-container
+ image: apache/submarine:mlflow-0.7.0-SNAPSHOT
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 5000
+ volumeMounts:
+ - mountPath: "/logs"
+ name: "volume"
+ subPath: "submarine-mlflow"
+ readinessProbe:
+ tcpSocket:
+ port: 5000
+ initialDelaySeconds: 60
+ periodSeconds: 10
+ volumes:
+ - name: "volume"
+ persistentVolumeClaim:
+ claimName: "submarine-mlflow-pvc"
+---
+# Source: submarine/templates/submarine-mlflow.yaml
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+ name: submarine-mlflow-ingressroute
+spec:
+ entryPoints:
+ - web
+ routes:
+ - kind: Rule
+ match: "PathPrefix(`/mlflow`)"
+ services:
+ - kind: Service
+ name: submarine-mlflow-service
+ port: 5000
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-rbac.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-rbac.yaml
new file mode 100644
index 0000000..2ea85a7
--- /dev/null
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-rbac.yaml
@@ -0,0 +1,86 @@
+---
+# Source: submarine/templates/rbac.yaml
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: "submarine-server"
+rules:
+- apiGroups:
+ - kubeflow.org
+ resources:
+ - tfjobs
+ - tfjobs/status
+ - pytorchjobs
+ - pytorchjobs/status
+ - notebooks
+ - notebooks/status
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - delete
+ - deletecollection
+ - patch
+ - update
+- apiGroups:
+ - traefik.containo.us
+ resources:
+ - ingressroutes
+ - middlewares
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - delete
+ - deletecollection
+ - patch
+ - update
+- apiGroups:
+ - ""
+ resources:
+ - pods
+ - pods/log
+ - services
+ - persistentvolumeclaims
+ - events
+ verbs:
+ - '*'
+- apiGroups:
+ - "apps"
+ resources:
+ - deployments
+ - deployments/status
+ verbs:
+ - '*'
+---
+# Source: submarine/templates/rbac.yaml
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: "submarine-server"
+subjects:
+- kind: ServiceAccount
+ name: "submarine-server"
+roleRef:
+ kind: Role
+ name: "submarine-server"
+ apiGroup: rbac.authorization.k8s.io
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-server.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-server.yaml
new file mode 100644
index 0000000..60ca20b
--- /dev/null
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-server.yaml
@@ -0,0 +1,90 @@
+---
+# Source: submarine/templates/submarine-server.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: "submarine-server"
+---
+# Source: submarine/templates/submarine-server.yaml
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: "submarine-server"
+ labels:
+ app: "submarine-server"
+spec:
+ ports:
+ - port: 8080
+ targetPort: 8080
+ protocol: TCP
+ selector:
+ app: "submarine-server"
+---
+# Source: submarine/templates/submarine-server.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: "submarine-server"
+spec:
+ selector:
+ matchLabels:
+ app: "submarine-server"
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: "submarine-server"
+
+ spec:
+ serviceAccountName: "submarine-server"
+ initContainers:
+ - name: submarine-server-initcontainer
+ image: "minio/mc"
+ command: ["/bin/bash", "-c",
+ "cnt=0;
+ while ! /bin/bash -c 'mc config host add minio
http://submarine-minio-service:9000
+ submarine_minio submarine_minio' 2>&1;
+ do
+ sleep 15;
+ ((cnt=cnt+1));
+ if [ $cnt -eq 80 ];then
+ echo 'ERROR: wait too long for minio pod';
+ exit 1;
+ fi;
+ done;
+ if /bin/bash -c 'mc ls minio/submarine' >/dev/null 2>&1; then
+ echo 'Bucket minio/submarine already exists, skipping creation.';
+ else
+ /bin/bash -c 'mc mb minio/submarine';
+ fi;"]
+ containers:
+ - name: "submarine-server"
+ env:
+ - name: SUBMARINE_SERVER_PORT
+ value: "8080"
+ - name: SUBMARINE_SERVER_PORT_8080_TCP
+ value: "8080"
+ - name: K8S_APISERVER_URL
+ value: "kubernetes.default.svc"
+
+ image: "apache/submarine:server-0.7.0-SNAPSHOT"
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 8080
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml
new file mode 100644
index 0000000..ae60e57
--- /dev/null
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml
@@ -0,0 +1,96 @@
+---
+# Source: submarine/templates/submarine-tensorboard.yaml
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Source: submarine/templates/submarine-tensorboard.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: submarine-tensorboard-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ storageClassName: "submarine-storageclass"
+ resources:
+ requests:
+ storage: "10Gi"
+---
+# Source: submarine/templates/submarine-tensorboard.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: submarine-tensorboard-service
+spec:
+ selector:
+ app: submarine-tensorboard
+ ports:
+ - protocol: TCP
+ port: 8080
+ targetPort: 6006
+---
+# Source: submarine/templates/submarine-tensorboard.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: submarine-tensorboard
+spec:
+ selector:
+ matchLabels:
+ app: submarine-tensorboard
+ template:
+ metadata:
+ labels:
+ app: submarine-tensorboard
+ spec:
+ containers:
+ - name: submarine-tensorboard-container
+ image: tensorflow/tensorflow:1.11.0
+ command:
+ - "tensorboard"
+ - "--logdir=/logs"
+ - "--path_prefix=/tensorboard"
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 6006
+ volumeMounts:
+ - mountPath: "/logs"
+ name: "volume"
+ subPath: "submarine-tensorboard"
+ readinessProbe:
+ tcpSocket:
+ port: 6006
+ periodSeconds: 10
+ volumes:
+ - name: "volume"
+ persistentVolumeClaim:
+ claimName: "submarine-tensorboard-pvc"
+---
+# Source: submarine/templates/submarine-tensorboard.yaml
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+ name: submarine-tensorboard-ingressroute
+spec:
+ entryPoints:
+ - web
+ routes:
+ - kind: Rule
+ match: "PathPrefix(`/tensorboard`)"
+ services:
+ - kind: Service
+ name: submarine-tensorboard-service
+ port: 8080
diff --git a/submarine-cloud-v2/dev.Dockerfile
b/submarine-cloud-v2/dev.Dockerfile
index 88d39f9..3fb3812 100644
--- a/submarine-cloud-v2/dev.Dockerfile
+++ b/submarine-cloud-v2/dev.Dockerfile
@@ -34,4 +34,6 @@ RUN curl -LO
https://dl.k8s.io/release/v1.14.2/bin/linux/amd64/kubectl &&\
kubectl version --client
ADD submarine-operator /usr/src
+COPY ["./artifacts", "/usr/src/artifacts"]
+
CMD ["/usr/src/submarine-operator", "-incluster=true"]
diff --git a/submarine-cloud-v2/pkg/controller/controller.go
b/submarine-cloud-v2/pkg/controller/controller.go
index ac8364b..08c7eb1 100644
--- a/submarine-cloud-v2/pkg/controller/controller.go
+++ b/submarine-cloud-v2/pkg/controller/controller.go
@@ -70,6 +70,14 @@ const (
minioPvcName = minioName + "-pvc"
minioServiceName = minioName + "-service"
minioIngressRouteName = minioName + "-ingressroute"
+ artifactPath = "./artifacts/submarine/"
+ databaseYamlPath = artifactPath + "submarine-database.yaml"
+ ingressYamlPath = artifactPath + "submarine-ingress.yaml"
+ minioYamlPath = artifactPath + "submarine-minio.yaml"
+ mlflowYamlPath = artifactPath + "submarine-mlflow.yaml"
+ serverYamlPath = artifactPath + "submarine-server.yaml"
+ tensorboardYamlPath = artifactPath +
"submarine-tensorboard.yaml"
+ rbacYamlPath = artifactPath + "submarine-rbac.yaml"
)
var dependents = []string{serverName, databaseName, tensorboardName,
mlflowName, minioName}
diff --git a/submarine-cloud-v2/pkg/controller/parser.go
b/submarine-cloud-v2/pkg/controller/parser.go
new file mode 100644
index 0000000..ca57093
--- /dev/null
+++ b/submarine-cloud-v2/pkg/controller/parser.go
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controller
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+
+ "github.com/pkg/errors"
+ traefikv1alpha1
"github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefik/v1alpha1"
+ appsv1 "k8s.io/api/apps/v1"
+ v1 "k8s.io/api/core/v1"
+ extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/util/yaml"
+)
+
+// PathToOSFile turn the file at the relativePath into a type of *os.File.
+func pathToOSFile(relativePath string) (*os.File, error) {
+ path, err := filepath.Abs(relativePath)
+ if err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("failed generate
absolute file path of %s", relativePath))
+ }
+
+ manifest, err := os.Open(path)
+ if err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("failed to open file
%s", path))
+ }
+
+ return manifest, nil
+}
+
+// ParseYaml
+func parseYaml(relativePath, kind string) ([]byte, error) {
+ var manifest *os.File
+ var err error
+
+ var marshaled []byte
+ if manifest, err = pathToOSFile(relativePath); err != nil {
+ return nil, err
+ }
+
+ decoder := yaml.NewYAMLOrJSONDecoder(manifest, 100)
+ for {
+ var out unstructured.Unstructured
+ err = decoder.Decode(&out)
+ if err != nil {
+ // this would indicate it's malformed YAML.
+ break
+ }
+
+ if out.GetKind() == kind {
+ marshaled, err = out.MarshalJSON()
+ break
+ }
+ }
+
+ if err != io.EOF && err != nil {
+ return nil, err
+ }
+ return marshaled, nil
+}
+
+// ParseServiceAccount parse ServiceAccount from yaml file.
+func ParseServiceAccountYaml(relativePath string) (*v1.ServiceAccount, error) {
+ var serviceAccount v1.ServiceAccount
+ marshaled, err := parseYaml(relativePath, "ServiceAccount")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &serviceAccount)
+ return &serviceAccount, nil
+}
+
+// ParseDeploymentYaml parse Deployment from yaml file.
+func ParseDeploymentYaml(relativePath string) (*appsv1.Deployment, error) {
+ var deployment appsv1.Deployment
+ marshaled, err := parseYaml(relativePath, "Deployment")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &deployment)
+ return &deployment, nil
+}
+
+// ParseServiceYaml parse Service from yaml file.
+func ParseServiceYaml(relativePath string) (*v1.Service, error) {
+ var service v1.Service
+ marshaled, err := parseYaml(relativePath, "Service")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &service)
+ return &service, nil
+}
+
+// ParseRoleBindingYaml parse RoleBinding from yaml file.
+func ParseRoleBindingYaml(relativePath string) (*rbacv1.RoleBinding, error) {
+ var rolebinding rbacv1.RoleBinding
+ marshaled, err := parseYaml(relativePath, "RoleBinding")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &rolebinding)
+ return &rolebinding, nil
+}
+
+// ParseRoleYaml parse Role from yaml file.
+func ParseRoleYaml(relativePath string) (*rbacv1.Role, error) {
+ var role rbacv1.Role
+ marshaled, err := parseYaml(relativePath, "Role")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &role)
+ return &role, nil
+}
+
+// ParseIngressYaml parse Ingress from yaml file.
+func ParseIngressYaml(relativePath string) (*extensionsv1beta1.Ingress, error)
{
+ var ingress extensionsv1beta1.Ingress
+ marshaled, err := parseYaml(relativePath, "Ingress")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &ingress)
+ return &ingress, nil
+}
+
+// ParseIngressYaml parse Ingress from yaml file.
+func ParsePersistentVolumeClaimYaml(relativePath string)
(*v1.PersistentVolumeClaim, error) {
+ var pvc v1.PersistentVolumeClaim
+ marshaled, err := parseYaml(relativePath, "PersistentVolumeClaim")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &pvc)
+ return &pvc, nil
+}
+
+// ParseIngressRouteYaml parse IngressRoute from yaml file.
+func ParseIngressRouteYaml(relativePath string)
(*traefikv1alpha1.IngressRoute, error) {
+ var ingressRoute traefikv1alpha1.IngressRoute
+ marshaled, err := parseYaml(relativePath, "IngressRoute")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &ingressRoute)
+ return &ingressRoute, nil
+}
diff --git a/submarine-cloud-v2/pkg/controller/submarine_database.go
b/submarine-cloud-v2/pkg/controller/submarine_database.go
index 275959d..af8857d 100644
--- a/submarine-cloud-v2/pkg/controller/submarine_database.go
+++ b/submarine-cloud-v2/pkg/controller/submarine_database.go
@@ -26,131 +26,46 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
- "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/klog/v2"
)
func newSubmarineDatabasePersistentVolumeClaim(submarine *v1alpha1.Submarine)
*corev1.PersistentVolumeClaim {
- storageClassName := storageClassName
- return &corev1.PersistentVolumeClaim{
- ObjectMeta: metav1.ObjectMeta{
- Name: databasePvcName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: corev1.PersistentVolumeClaimSpec{
- AccessModes: []corev1.PersistentVolumeAccessMode{
- corev1.ReadWriteOnce,
- },
- Resources: corev1.ResourceRequirements{
- Requests: corev1.ResourceList{
- corev1.ResourceStorage:
resource.MustParse(submarine.Spec.Database.StorageSize),
- },
- },
- StorageClassName: &storageClassName,
- },
+ pvc, err := ParsePersistentVolumeClaimYaml(databaseYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParsePersistentVolumeClaim", err)
+ }
+ pvc.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+
+ return pvc
}
func newSubmarineDatabaseDeployment(submarine *v1alpha1.Submarine)
*appsv1.Deployment {
- databaseImage := submarine.Spec.Database.Image
- if databaseImage == "" {
- databaseImage = "apache/submarine:database-" +
submarine.Spec.Version
- }
-
- return &appsv1.Deployment{
- ObjectMeta: metav1.ObjectMeta{
- Name: databaseName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: appsv1.DeploymentSpec{
- Selector: &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app": databaseName,
- },
- },
- Replicas: submarine.Spec.Database.Replicas,
- Template: corev1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app": databaseName,
- },
- },
- Spec: corev1.PodSpec{
- Containers: []corev1.Container{
- {
- Name:
databaseName,
- Image:
databaseImage,
- ImagePullPolicy:
"IfNotPresent",
- Ports:
[]corev1.ContainerPort{
- {
-
ContainerPort: databasePort,
- },
- },
- Env: []corev1.EnvVar{
- {
- Name:
"MYSQL_ROOT_PASSWORD",
- Value:
"password",
- },
- },
- VolumeMounts:
[]corev1.VolumeMount{
- {
-
MountPath: "/var/lib/mysql",
- Name:
"volume",
-
SubPath: databaseName,
- },
- },
- ReadinessProbe:
&corev1.Probe{
- Handler:
corev1.Handler{
-
TCPSocket: &corev1.TCPSocketAction{
-
Port: intstr.FromInt(databasePort),
- },
- },
- },
- },
- },
- Volumes: []corev1.Volume{
- {
- Name: "volume",
- VolumeSource:
corev1.VolumeSource{
-
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-
ClaimName: databasePvcName,
- },
- },
- },
- },
- },
- },
- },
+ databaseReplicas := *submarine.Spec.Database.Replicas
+
+ deployment, err := ParseDeploymentYaml(databaseYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseDeploymentYaml", err)
}
+ deployment.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
+ }
+ deployment.Spec.Replicas = &databaseReplicas
+
+ return deployment
}
func newSubmarineDatabaseService(submarine *v1alpha1.Submarine)
*corev1.Service {
- return &corev1.Service{
- ObjectMeta: metav1.ObjectMeta{
- Name: databaseName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: corev1.ServiceSpec{
- Ports: []corev1.ServicePort{
- {
- Port: databasePort,
- TargetPort:
intstr.FromInt(databasePort),
- Name: databaseName,
- },
- },
- Selector: map[string]string{
- "app": databaseName,
- },
- },
+ service, err := ParseServiceYaml(databaseYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseServiceYaml", err)
+ }
+ service.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+ return service
}
// createSubmarineDatabase is a function to create submarine-database.
diff --git a/submarine-cloud-v2/pkg/controller/submarine_ingress.go
b/submarine-cloud-v2/pkg/controller/submarine_ingress.go
index 8de307e..375f9f0 100644
--- a/submarine-cloud-v2/pkg/controller/submarine_ingress.go
+++ b/submarine-cloud-v2/pkg/controller/submarine_ingress.go
@@ -27,39 +27,20 @@ import (
extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/klog/v2"
)
func newSubmarineServerIngress(submarine *v1alpha1.Submarine)
*extensionsv1beta1.Ingress {
- return &extensionsv1beta1.Ingress{
- ObjectMeta: metav1.ObjectMeta{
- Name: ingressName,
- Namespace: submarine.Namespace,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: extensionsv1beta1.IngressSpec{
- Rules: []extensionsv1beta1.IngressRule{
- {
- IngressRuleValue:
extensionsv1beta1.IngressRuleValue{
- HTTP:
&extensionsv1beta1.HTTPIngressRuleValue{
- Paths:
[]extensionsv1beta1.HTTPIngressPath{
- {
-
Backend: extensionsv1beta1.IngressBackend{
-
ServiceName: serverName,
-
ServicePort: intstr.FromInt(8080),
- },
- Path:
"/",
- },
- },
- },
- },
- },
- },
- },
+ ingress, err := ParseIngressYaml(ingressYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseIngressYaml", err)
+ }
+
+ ingress.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+
+ return ingress
}
// createIngress is a function to create Ingress.
diff --git a/submarine-cloud-v2/pkg/controller/submarine_minio.go
b/submarine-cloud-v2/pkg/controller/submarine_minio.go
index 0fb6309..c38650c 100644
--- a/submarine-cloud-v2/pkg/controller/submarine_minio.go
+++ b/submarine-cloud-v2/pkg/controller/submarine_minio.go
@@ -26,161 +26,56 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
- "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/klog/v2"
traefikv1alpha1
"github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefik/v1alpha1"
)
func newSubmarineMinioPersistentVolumeClaim(submarine *v1alpha1.Submarine)
*corev1.PersistentVolumeClaim {
- storageClassName := storageClassName
- return &corev1.PersistentVolumeClaim{
- ObjectMeta: metav1.ObjectMeta{
- Name: minioPvcName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: corev1.PersistentVolumeClaimSpec{
- AccessModes: []corev1.PersistentVolumeAccessMode{
- corev1.ReadWriteOnce,
- },
- Resources: corev1.ResourceRequirements{
- Requests: corev1.ResourceList{
- corev1.ResourceStorage:
resource.MustParse(submarine.Spec.Minio.StorageSize),
- },
- },
- StorageClassName: &storageClassName,
- },
+ pvc, err := ParsePersistentVolumeClaimYaml(minioYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParsePersistentVolumeClaim", err)
+ }
+ pvc.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+
+ return pvc
}
func newSubmarineMinioDeployment(submarine *v1alpha1.Submarine)
*appsv1.Deployment {
- return &appsv1.Deployment{
- ObjectMeta: metav1.ObjectMeta{
- Name: minioName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: appsv1.DeploymentSpec{
- Selector: &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app": minioName,
- },
- },
- Template: corev1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app": minioName,
- },
- },
- Spec: corev1.PodSpec{
- Containers: []corev1.Container{
- {
- Name:
minioName + "-container",
- Image:
"minio/minio:latest",
- ImagePullPolicy:
"IfNotPresent",
- Args: []string{
- "server",
- "/data",
- },
- Env: []corev1.EnvVar{
- {
- Name:
"MINIO_ACCESS_KEY",
- Value:
"submarine_minio",
- },
- {
- Name:
"MINIO_SECRET_KEY",
- Value:
"submarine_minio",
- },
- },
- // TODO env
- Ports:
[]corev1.ContainerPort{
- {
-
ContainerPort: 9000,
- },
- },
- VolumeMounts:
[]corev1.VolumeMount{
- {
-
MountPath: "/data",
- Name:
"volume",
-
SubPath: minioName,
- },
- },
- },
- },
- Volumes: []corev1.Volume{
- {
- Name: "volume",
- VolumeSource:
corev1.VolumeSource{
-
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-
ClaimName: minioPvcName,
- },
- },
- },
- },
- },
- },
- },
+ deployment, err := ParseDeploymentYaml(minioYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseDeploymentYaml", err)
+ }
+ deployment.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+
+ return deployment
}
func newSubmarineMinioService(submarine *v1alpha1.Submarine) *corev1.Service {
- return &corev1.Service{
- ObjectMeta: metav1.ObjectMeta{
- Name: minioServiceName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: corev1.ServiceSpec{
- Type: corev1.ServiceTypeClusterIP,
- Selector: map[string]string{
- "app": minioName,
- },
- Ports: []corev1.ServicePort{
- {
- Protocol: "TCP",
- Port: 9000,
- TargetPort: intstr.FromInt(9000),
- },
- },
- },
+ service, err := ParseServiceYaml(minioYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseServiceYaml", err)
+ }
+ service.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+ return service
}
func newSubmarineMinioIngressRoute(submarine *v1alpha1.Submarine)
*traefikv1alpha1.IngressRoute {
- return &traefikv1alpha1.IngressRoute{
- ObjectMeta: metav1.ObjectMeta{
- Name: minioIngressRouteName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: traefikv1alpha1.IngressRouteSpec{
- EntryPoints: []string{
- "web",
- },
- Routes: []traefikv1alpha1.Route{
- {
- Kind: "Rule",
- Match: "PathPrefix(`/minio`)",
- Services: []traefikv1alpha1.Service{
- {
- LoadBalancerSpec:
traefikv1alpha1.LoadBalancerSpec{
- Kind: "Service",
- Name:
minioServiceName,
- Port: 9000,
- },
- },
- },
- },
- },
- },
+ ingressRoute, err := ParseIngressRouteYaml(minioYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseIngressRouteYaml", err)
+ }
+ ingressRoute.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+ return ingressRoute
}
// createSubmarineMinio is a function to create submarine-minio.
diff --git a/submarine-cloud-v2/pkg/controller/submarine_mlflow.go
b/submarine-cloud-v2/pkg/controller/submarine_mlflow.go
index 93d7d38..c5c963d 100644
--- a/submarine-cloud-v2/pkg/controller/submarine_mlflow.go
+++ b/submarine-cloud-v2/pkg/controller/submarine_mlflow.go
@@ -26,164 +26,54 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
- "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/klog/v2"
)
func newSubmarineMlflowPersistentVolumeClaim(submarine *v1alpha1.Submarine)
*corev1.PersistentVolumeClaim {
- storageClassName := storageClassName
- return &corev1.PersistentVolumeClaim{
- ObjectMeta: metav1.ObjectMeta{
- Name: mlflowPvcName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: corev1.PersistentVolumeClaimSpec{
- AccessModes: []corev1.PersistentVolumeAccessMode{
- corev1.ReadWriteOnce,
- },
- Resources: corev1.ResourceRequirements{
- Requests: corev1.ResourceList{
- corev1.ResourceStorage:
resource.MustParse(submarine.Spec.Mlflow.StorageSize),
- },
- },
- StorageClassName: &storageClassName,
- },
+ pvc, err := ParsePersistentVolumeClaimYaml(mlflowYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParsePersistentVolumeClaim", err)
+ }
+ pvc.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+
+ return pvc
}
func newSubmarineMlflowDeployment(submarine *v1alpha1.Submarine)
*appsv1.Deployment {
- return &appsv1.Deployment{
- ObjectMeta: metav1.ObjectMeta{
- Name: mlflowName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: appsv1.DeploymentSpec{
- Selector: &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app": mlflowName,
- },
- },
- Template: corev1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app": mlflowName,
- },
- },
- Spec: corev1.PodSpec{
- InitContainers: []corev1.Container{
- {
- Name:
"check-database-connection",
- Image: "busybox:1.28",
- Command: []string{
- "sh",
- "-c",
-
fmt.Sprintf("until nc -z %s %d; do echo waiting for database connection; sleep
20; done", databaseName, databasePort),
- },
- },
- },
- Containers: []corev1.Container{
- {
- Name:
mlflowName + "-container",
- Image:
"apache/submarine:mlflow-0.7.0-SNAPSHOT",
- ImagePullPolicy:
"IfNotPresent",
- Ports:
[]corev1.ContainerPort{
- {
-
ContainerPort: 5000,
- },
- },
- VolumeMounts:
[]corev1.VolumeMount{
- {
-
MountPath: "/logs",
- Name:
"volume",
-
SubPath: mlflowName,
- },
- },
- ReadinessProbe:
&corev1.Probe{
- Handler:
corev1.Handler{
-
TCPSocket: &corev1.TCPSocketAction{
-
Port: intstr.FromInt(5000),
- },
- },
-
InitialDelaySeconds: 60,
- PeriodSeconds:
10,
- },
- },
- },
- Volumes: []corev1.Volume{
- {
- Name: "volume",
- VolumeSource:
corev1.VolumeSource{
-
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-
ClaimName: mlflowPvcName,
- },
- },
- },
- },
- },
- },
- },
+ deployment, err := ParseDeploymentYaml(mlflowYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseDeploymentYaml", err)
+ }
+ deployment.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+
+ return deployment
}
func newSubmarineMlflowService(submarine *v1alpha1.Submarine) *corev1.Service {
- return &corev1.Service{
- ObjectMeta: metav1.ObjectMeta{
- Name: mlflowServiceName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: corev1.ServiceSpec{
- Type: corev1.ServiceTypeClusterIP,
- Selector: map[string]string{
- "app": mlflowName,
- },
- Ports: []corev1.ServicePort{
- {
- Protocol: "TCP",
- Port: 5000,
- TargetPort: intstr.FromInt(5000),
- },
- },
- },
+ service, err := ParseServiceYaml(mlflowYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseServiceYaml", err)
+ }
+ service.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+ return service
}
func newSubmarineMlflowIngressRoute(submarine *v1alpha1.Submarine)
*traefikv1alpha1.IngressRoute {
- return &traefikv1alpha1.IngressRoute{
- ObjectMeta: metav1.ObjectMeta{
- Name: mlflowName + "-ingressroute",
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: traefikv1alpha1.IngressRouteSpec{
- EntryPoints: []string{
- "web",
- },
- Routes: []traefikv1alpha1.Route{
- {
- Kind: "Rule",
- Match: "PathPrefix(`/mlflow`)",
- Services: []traefikv1alpha1.Service{
- {
- LoadBalancerSpec:
traefikv1alpha1.LoadBalancerSpec{
- Kind: "Service",
- Name:
mlflowServiceName,
- Port: 5000,
- },
- },
- },
- },
- },
- },
+ ingressRoute, err := ParseIngressRouteYaml(mlflowYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseIngressRouteYaml", err)
+ }
+ ingressRoute.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+ return ingressRoute
}
// createSubmarineMlflow is a function to create submarine-mlflow.
diff --git a/submarine-cloud-v2/pkg/controller/submarine_server.go
b/submarine-cloud-v2/pkg/controller/submarine_server.go
index b2c9988..9ee86ea 100644
--- a/submarine-cloud-v2/pkg/controller/submarine_server.go
+++ b/submarine-cloud-v2/pkg/controller/submarine_server.go
@@ -27,132 +27,75 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/klog/v2"
)
func newSubmarineServerServiceAccount(submarine *v1alpha1.Submarine)
*corev1.ServiceAccount {
- return &corev1.ServiceAccount{
- ObjectMeta: metav1.ObjectMeta{
- Name: serverName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
+ serviceAccount, err := ParseServiceAccountYaml(serverYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseServiceAccountYaml", err)
}
+
+ serviceAccount.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
+ }
+
+ return serviceAccount
}
func newSubmarineServerService(submarine *v1alpha1.Submarine) *corev1.Service {
- return &corev1.Service{
- ObjectMeta: metav1.ObjectMeta{
- Name: serverName,
- Labels: map[string]string{
- "app": serverName,
- },
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: corev1.ServiceSpec{
- Ports: []corev1.ServicePort{
- {
- Port: 8080,
- TargetPort: intstr.FromInt(8080),
- Protocol: "TCP",
- },
- },
- Selector: map[string]string{
- "app": serverName,
- },
- },
+ service, err := ParseServiceYaml(serverYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseServiceYaml", err)
}
+ service.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
+ }
+ return service
}
func newSubmarineServerDeployment(submarine *v1alpha1.Submarine)
*appsv1.Deployment {
- serverImage := submarine.Spec.Server.Image
serverReplicas := *submarine.Spec.Server.Replicas
- if serverImage == "" {
- serverImage = "apache/submarine:server-" +
submarine.Spec.Version
- }
ownerReference := *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine"))
-
- return &appsv1.Deployment{
- ObjectMeta: metav1.ObjectMeta{
- Name: serverName,
- OwnerReferences: []metav1.OwnerReference{
- ownerReference,
- },
+ operatorEnv := []corev1.EnvVar{
+ {
+ Name: "SUBMARINE_SERVER_DNS_NAME",
+ Value: serverName + "." + submarine.Namespace,
+ },
+ {
+ Name: "ENV_NAMESPACE",
+ Value: submarine.Namespace,
+ },
+ {
+ Name: "SUBMARINE_APIVERSION",
+ Value: ownerReference.APIVersion,
+ },
+ {
+ Name: "SUBMARINE_KIND",
+ Value: ownerReference.Kind,
+ },
+ {
+ Name: "SUBMARINE_NAME",
+ Value: ownerReference.Name,
},
- Spec: appsv1.DeploymentSpec{
- Selector: &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app": serverName,
- },
- },
- Replicas: &serverReplicas,
- Template: corev1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app": serverName,
- },
- },
- Spec: corev1.PodSpec{
- ServiceAccountName: serverName,
- Containers: []corev1.Container{
- {
- Name: serverName,
- Image: serverImage,
- Env: []corev1.EnvVar{
- {
- Name:
"SUBMARINE_SERVER_PORT",
- Value:
"8080",
- },
- {
- Name:
"SUBMARINE_SERVER_PORT_8080_TCP",
- Value:
"8080",
- },
- {
- Name:
"SUBMARINE_SERVER_DNS_NAME",
- Value:
serverName + "." + submarine.Namespace,
- },
- {
- Name:
"K8S_APISERVER_URL",
- Value:
"kubernetes.default.svc",
- },
- {
- Name:
"ENV_NAMESPACE",
- Value:
submarine.Namespace,
- },
- {
- Name:
"SUBMARINE_APIVERSION",
- Value:
ownerReference.APIVersion,
- },
- {
- Name:
"SUBMARINE_KIND",
- Value:
ownerReference.Kind,
- },
- {
- Name:
"SUBMARINE_NAME",
- Value:
ownerReference.Name,
- },
- {
- Name:
"SUBMARINE_UID",
- Value:
string(ownerReference.UID),
- },
- },
- Ports:
[]corev1.ContainerPort{
- {
-
ContainerPort: 8080,
- },
- },
- ImagePullPolicy:
"IfNotPresent",
- },
- },
- },
- },
+ {
+ Name: "SUBMARINE_UID",
+ Value: string(ownerReference.UID),
},
}
+
+ deployment, err := ParseDeploymentYaml(serverYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseDeploymentYaml", err)
+ }
+ deployment.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ ownerReference,
+ }
+ deployment.Spec.Replicas = &serverReplicas
+ deployment.Spec.Template.Spec.Containers[0].Env =
append(deployment.Spec.Template.Spec.Containers[0].Env, operatorEnv...)
+
+ return deployment
}
// createSubmarineServer is a function to create submarine-server.
diff --git a/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go
b/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go
index 2b92b1e..61e0bb8 100644
--- a/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go
+++ b/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go
@@ -31,59 +31,27 @@ import (
)
func newSubmarineServerRole(submarine *v1alpha1.Submarine) *rbacv1.Role {
- return &rbacv1.Role{
- ObjectMeta: metav1.ObjectMeta{
- Name: serverName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Rules: []rbacv1.PolicyRule{
- {
- Verbs: []string{"get", "list", "watch",
"create", "delete", "deletecollection", "patch", "update"},
- APIGroups: []string{"kubeflow.org"},
- Resources: []string{"tfjobs", "tfjobs/status",
"pytorchjobs", "pytorchjobs/status", "notebooks", "notebooks/status"},
- },
- {
- Verbs: []string{"get", "list", "watch",
"create", "delete", "deletecollection", "patch", "update"},
- APIGroups: []string{"traefik.containo.us"},
- Resources: []string{"ingressroutes"},
- },
- {
- Verbs: []string{"*"},
- APIGroups: []string{""},
- Resources: []string{"pods", "pods/log",
"services", "persistentvolumes", "persistentvolumeclaims"},
- },
- {
- Verbs: []string{"*"},
- APIGroups: []string{"apps"},
- Resources: []string{"deployments",
"deployments/status"},
- },
- },
+ role, err := ParseRoleYaml(rbacYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseRole", err)
+ }
+ role.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+
+ return role
}
func newSubmarineServerRoleBinding(submarine *v1alpha1.Submarine)
*rbacv1.RoleBinding {
- return &rbacv1.RoleBinding{
- ObjectMeta: metav1.ObjectMeta{
- Name: serverName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Subjects: []rbacv1.Subject{
- {
- Kind: "ServiceAccount",
- Namespace: submarine.Namespace,
- Name: serverName,
- },
- },
- RoleRef: rbacv1.RoleRef{
- Kind: "Role",
- Name: serverName,
- APIGroup: "rbac.authorization.k8s.io",
- },
+ roleBinding, err := ParseRoleBindingYaml(rbacYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseRoleBinding", err)
+ }
+ roleBinding.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+
+ return roleBinding
}
// createSubmarineServerRBAC is a function to create RBAC for submarine-server.
diff --git a/submarine-cloud-v2/pkg/controller/submarine_tensorboard.go
b/submarine-cloud-v2/pkg/controller/submarine_tensorboard.go
index 4b73979..b8b29f3 100644
--- a/submarine-cloud-v2/pkg/controller/submarine_tensorboard.go
+++ b/submarine-cloud-v2/pkg/controller/submarine_tensorboard.go
@@ -26,158 +26,56 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
- "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/klog/v2"
traefikv1alpha1
"github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefik/v1alpha1"
)
func newSubmarineTensorboardPersistentVolumeClaim(submarine
*v1alpha1.Submarine) *corev1.PersistentVolumeClaim {
- storageClassName := storageClassName
- return &corev1.PersistentVolumeClaim{
- ObjectMeta: metav1.ObjectMeta{
- Name: tensorboardPvcName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: corev1.PersistentVolumeClaimSpec{
- AccessModes: []corev1.PersistentVolumeAccessMode{
- corev1.ReadWriteOnce,
- },
- Resources: corev1.ResourceRequirements{
- Requests: corev1.ResourceList{
- corev1.ResourceStorage:
resource.MustParse(submarine.Spec.Tensorboard.StorageSize),
- },
- },
- StorageClassName: &storageClassName,
- },
+ pvc, err := ParsePersistentVolumeClaimYaml(tensorboardYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParsePersistentVolumeClaim", err)
+ }
+ pvc.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+
+ return pvc
}
func newSubmarineTensorboardDeployment(submarine *v1alpha1.Submarine)
*appsv1.Deployment {
- return &appsv1.Deployment{
- ObjectMeta: metav1.ObjectMeta{
- Name: tensorboardName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: appsv1.DeploymentSpec{
- Selector: &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app": tensorboardName,
- },
- },
- Template: corev1.PodTemplateSpec{
- ObjectMeta: metav1.ObjectMeta{
- Labels: map[string]string{
- "app": tensorboardName,
- },
- },
- Spec: corev1.PodSpec{
- Containers: []corev1.Container{
- {
- Name: tensorboardName
+ "-container",
- Image:
"tensorflow/tensorflow:1.11.0",
- Command: []string{
- "tensorboard",
-
"--logdir=/logs",
-
"--path_prefix=/tensorboard",
- },
- ImagePullPolicy:
"IfNotPresent",
- Ports:
[]corev1.ContainerPort{
- {
-
ContainerPort: 6006,
- },
- },
- VolumeMounts:
[]corev1.VolumeMount{
- {
-
MountPath: "/logs",
- Name:
"volume",
-
SubPath: tensorboardName,
- },
- },
- ReadinessProbe:
&corev1.Probe{
- Handler:
corev1.Handler{
-
TCPSocket: &corev1.TCPSocketAction{
-
Port: intstr.FromInt(6006),
- },
- },
- PeriodSeconds:
10,
- },
- },
- },
- Volumes: []corev1.Volume{
- {
- Name: "volume",
- VolumeSource:
corev1.VolumeSource{
-
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-
ClaimName: tensorboardPvcName,
- },
- },
- },
- },
- },
- },
- },
+ deployment, err := ParseDeploymentYaml(tensorboardYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseDeploymentYaml", err)
+ }
+ deployment.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+
+ return deployment
}
func newSubmarineTensorboardService(submarine *v1alpha1.Submarine)
*corev1.Service {
- return &corev1.Service{
- ObjectMeta: metav1.ObjectMeta{
- Name: tensorboardServiceName,
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: corev1.ServiceSpec{
- Selector: map[string]string{
- "app": tensorboardName,
- },
- Ports: []corev1.ServicePort{
- {
- Protocol: "TCP",
- Port: 8080,
- TargetPort: intstr.FromInt(6006),
- },
- },
- },
+ service, err := ParseServiceYaml(tensorboardYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseServiceYaml", err)
+ }
+ service.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+ return service
}
func newSubmarineTensorboardIngressRoute(submarine *v1alpha1.Submarine)
*traefikv1alpha1.IngressRoute {
- return &traefikv1alpha1.IngressRoute{
- ObjectMeta: metav1.ObjectMeta{
- Name: tensorboardName + "-ingressroute",
- OwnerReferences: []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- },
- },
- Spec: traefikv1alpha1.IngressRouteSpec{
- EntryPoints: []string{
- "web",
- },
- Routes: []traefikv1alpha1.Route{
- {
- Kind: "Rule",
- Match: "PathPrefix(`/tensorboard`)",
- Services: []traefikv1alpha1.Service{
- {
- LoadBalancerSpec:
traefikv1alpha1.LoadBalancerSpec{
- Kind: "Service",
- Name:
tensorboardServiceName,
- Port: 8080,
- },
- },
- },
- },
- },
- },
+ ingressRoute, err := ParseIngressRouteYaml(tensorboardYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseIngressRouteYaml", err)
+ }
+ ingressRoute.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+ return ingressRoute
}
// createSubmarineTensorboard is a function to create submarine-tensorboard.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
