This is an automated email from the ASF dual-hosted git repository.
chishengliu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new d47d3a84 SUBMARINE-1275. Submarine-cloud-v3 reconcile logic
d47d3a84 is described below
commit d47d3a84f766819af832332ec92566f104efcfae
Author: joshvictor1024 <[email protected]>
AuthorDate: Wed Jun 1 16:48:09 2022 +0800
SUBMARINE-1275. Submarine-cloud-v3 reconcile logic
### What is this PR for?
Add reconcile logic to submarine-cloud-v3
### What type of PR is it?
Feature
### Todos
* [x] - add reconcile logic
* [x] - update cluster role rules
* [x] - add new development doc
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-1275
### How should this be tested?
Follow the `submarine-cloud-v3/docs/developer-guide.md`
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? Yes
* Does this need new documentation? Yes
Author: joshvictor1024 <[email protected]>
Signed-off-by: Chi-Sheng Liu <[email protected]>
Closes #959 from joshvictor1024/SUBMARINE-1275 and squashes the following
commits:
76a53ccd [joshvictor1024] docs: fix link, add k8s version
8ccd557b [joshvictor1024] update development doc
54ababd8 [joshvictor1024] add reconcile logic
76a2fbfb [joshvictor1024] add code to create resources from yaml
7a8be59f [joshvictor1024] add cluster role rules. regenerate yaml.
aa4e7ecc [joshvictor1024] add artifacts
d29c74f1 [joshvictor1024] add event recorder and logger
156a3e50 [joshvictor1024] add submarine flags
---
submarine-cloud-v2/docs/developer-guide.md | 4 +-
.../artifacts/submarine-database.yaml | 80 +++++
submarine-cloud-v3/artifacts/submarine-minio.yaml | 80 +++++
submarine-cloud-v3/artifacts/submarine-mlflow.yaml | 108 ++++++
.../submarine-observer-rbac.yaml} | 49 +--
.../artifacts/submarine-server-rbac.yaml | 142 ++++++++
submarine-cloud-v3/artifacts/submarine-server.yaml | 99 ++++++
.../submarine-storage-rbac.yaml} | 50 ++-
.../artifacts/submarine-tensorboard.yaml | 79 +++++
.../artifacts/submarine-virtualservice.yaml | 57 ++++
submarine-cloud-v3/config/rbac/role.yaml | 103 ++++++
submarine-cloud-v3/controllers/parser.go | 168 +++++++++
.../controllers/submarine_controller.go | 375 ++++++++++++++++++++-
.../controllers/submarine_database.go | 160 +++++++++
submarine-cloud-v3/controllers/submarine_minio.go | 148 ++++++++
submarine-cloud-v3/controllers/submarine_mlflow.go | 151 +++++++++
.../controllers/submarine_observer_rbac.go | 113 +++++++
submarine-cloud-v3/controllers/submarine_server.go | 200 +++++++++++
.../controllers/submarine_server_rbac.go | 123 +++++++
.../controllers/submarine_storage_rbac.go | 155 +++++++++
.../controllers/submarine_tensorboard.go | 151 +++++++++
.../controllers/submarine_virtualservice.go | 77 +++++
submarine-cloud-v3/docs/developer-guide.md | 65 +++-
submarine-cloud-v3/go.mod | 15 +-
submarine-cloud-v3/go.sum | 24 +-
submarine-cloud-v3/main.go | 40 ++-
26 files changed, 2731 insertions(+), 85 deletions(-)
diff --git a/submarine-cloud-v2/docs/developer-guide.md
b/submarine-cloud-v2/docs/developer-guide.md
index 2e0b3212..a0df1baf 100644
--- a/submarine-cloud-v2/docs/developer-guide.md
+++ b/submarine-cloud-v2/docs/developer-guide.md
@@ -21,7 +21,7 @@ Golang version: `1.17`
## Prerequisites
-First finish the prerequisites specified in the [QuickStart](quickstart)
section on the submarine website.
+First finish the prerequisites specified in the
[QuickStart](https://submarine.apache.org/docs/next/gettingStarted/quickstart)
section on the submarine website.
Next, install golang dependencies.
@@ -31,7 +31,7 @@ go mod vendor
## Run operator in-cluster
-If you follow the [QuickStart][quickstart] section on the submarine website,
you are running operator in-cluster.
+If you follow the
[QuickStart](https://submarine.apache.org/docs/next/gettingStarted/quickstart)
section on the submarine website, you are running operator in-cluster.
## Run operator out-of-cluster
diff --git a/submarine-cloud-v3/artifacts/submarine-database.yaml
b/submarine-cloud-v3/artifacts/submarine-database.yaml
new file mode 100644
index 00000000..9800a2d4
--- /dev/null
+++ b/submarine-cloud-v3/artifacts/submarine-database.yaml
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: submarine-database-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ storageClassName: submarine-storageclass
+ resources:
+ requests:
+ storage: 1Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: "submarine-database"
+spec:
+ ports:
+ - name: "submarine-database"
+ port: 3306
+ targetPort: 3306
+ clusterIP: None
+ type: ClusterIP
+ selector:
+ app: "submarine-database"
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: "submarine-database"
+spec:
+ serviceName: submarine-database
+ replicas: 1
+ selector:
+ matchLabels:
+ app: "submarine-database"
+ template:
+ metadata:
+ labels:
+ app: "submarine-database"
+ spec:
+ serviceAccountName: "submarine-storage"
+ containers:
+ - name: "submarine-database"
+ image: "apache/submarine:database-0.8.0-SNAPSHOT"
+ imagePullPolicy: "IfNotPresent"
+ ports:
+ - containerPort: 3306
+ env:
+ - name: MYSQL_ROOT_PASSWORD
+ value: "password"
+ volumeMounts:
+ - mountPath: /var/lib/mysql
+ name: volume
+ subPath: submarine-database
+ readinessProbe:
+ tcpSocket:
+ port: 3306
+ volumes:
+ - name: volume
+ persistentVolumeClaim:
+ claimName: submarine-database-pvc
diff --git a/submarine-cloud-v3/artifacts/submarine-minio.yaml
b/submarine-cloud-v3/artifacts/submarine-minio.yaml
new file mode 100644
index 00000000..8cf62450
--- /dev/null
+++ b/submarine-cloud-v3/artifacts/submarine-minio.yaml
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: submarine-minio-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ storageClassName: "submarine-storageclass"
+ resources:
+ requests:
+ storage: "10Gi"
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: submarine-minio-service
+spec:
+ type: ClusterIP
+ selector:
+ app: submarine-minio
+ ports:
+ - protocol: TCP
+ port: 9000
+ targetPort: 9000
+ name: http
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: submarine-minio
+spec:
+ selector:
+ matchLabels:
+ app: submarine-minio
+ template:
+ metadata:
+ labels:
+ app: submarine-minio
+ spec:
+ serviceAccountName: "submarine-storage"
+ containers:
+ - name: submarine-minio-container
+ image: minio/minio:RELEASE.2021-02-14T04-01-33Z
+ imagePullPolicy: IfNotPresent
+ args:
+ - server
+ - /data
+ env:
+ - name: MINIO_ACCESS_KEY
+ value: "submarine_minio"
+ - name: MINIO_SECRET_KEY
+ value: "submarine_minio"
+ ports:
+ - containerPort: 9000
+ volumeMounts:
+ - mountPath: "/data"
+ name: "volume"
+ subPath: "submarine-minio"
+ volumes:
+ - name: "volume"
+ persistentVolumeClaim:
+ claimName: "submarine-minio-pvc"
diff --git a/submarine-cloud-v3/artifacts/submarine-mlflow.yaml
b/submarine-cloud-v3/artifacts/submarine-mlflow.yaml
new file mode 100644
index 00000000..71091016
--- /dev/null
+++ b/submarine-cloud-v3/artifacts/submarine-mlflow.yaml
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: submarine-mlflow-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ storageClassName: "submarine-storageclass"
+ resources:
+ requests:
+ storage: "10Gi"
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: submarine-mlflow-service
+spec:
+ type: ClusterIP
+ selector:
+ app: submarine-mlflow
+ ports:
+ - protocol: TCP
+ port: 5000
+ targetPort: 5000
+ name: http
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: submarine-mlflow
+spec:
+ selector:
+ matchLabels:
+ app: submarine-mlflow
+ template:
+ metadata:
+ labels:
+ app: submarine-mlflow
+ spec:
+ serviceAccountName: "submarine-storage"
+ initContainers:
+ - name: check-database-connection
+ image: busybox:1.28
+ command: ["sh", "-c",
+ "until nc -z submarine-database 3306;
+ do echo waiting for database connection;
+ sleep 20; done"]
+ - name: submarine-mlflow-initcontainer
+ image: "minio/mc"
+ command: ["/bin/bash", "-c",
+ "cnt=0;
+ while ! /bin/bash -c 'mc --config-dir /root/.mc config host add minio
http://submarine-minio-service:9000
+ submarine_minio submarine_minio' 2>&1;
+ do
+ sleep 15;
+ ((cnt=cnt+1));
+ if [ $cnt -eq 80 ];then
+ echo 'ERROR: wait too long for minio pod';
+ exit 1;
+ fi;
+ done;
+ if /bin/bash -c 'mc --config-dir /root/.mc ls minio/mlflow' >/dev/null
2>&1; then
+ echo 'Bucket minio/mlflow already exists, skipping creation.';
+ else
+ /bin/bash -c 'mc --config-dir /root/.mc mb minio/mlflow';
+ fi;"]
+ volumeMounts:
+ - name: mc-config-vol
+ mountPath: /root/.mc
+ containers:
+ - name: submarine-mlflow-container
+ image: apache/submarine:mlflow-0.8.0-SNAPSHOT
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 5000
+ volumeMounts:
+ - mountPath: "/logs"
+ name: "volume"
+ subPath: "submarine-mlflow"
+ readinessProbe:
+ tcpSocket:
+ port: 5000
+ initialDelaySeconds: 60
+ periodSeconds: 10
+ volumes:
+ - name: "volume"
+ persistentVolumeClaim:
+ claimName: "submarine-mlflow-pvc"
+ - name: mc-config-vol
+ emptyDir: {}
diff --git a/submarine-cloud-v3/config/rbac/role.yaml
b/submarine-cloud-v3/artifacts/submarine-observer-rbac.yaml
similarity index 64%
copy from submarine-cloud-v3/config/rbac/role.yaml
copy to submarine-cloud-v3/artifacts/submarine-observer-rbac.yaml
index e73b5772..60771481 100644
--- a/submarine-cloud-v3/config/rbac/role.yaml
+++ b/submarine-cloud-v3/artifacts/submarine-observer-rbac.yaml
@@ -17,34 +17,45 @@
---
apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
metadata:
- creationTimestamp: null
- name: manager-role
+ name: "submarine-observer"
rules:
- apiGroups:
- - submarine.apache.org
+ - kubeflow.org
resources:
- - submarines
+ - tfjobs
+ - tfjobs/status
+ - pytorchjobs
+ - pytorchjobs/status
+ - notebooks
+ - notebooks/status
verbs:
- - create
- - delete
- get
- list
- - patch
- - update
- watch
- apiGroups:
- - submarine.apache.org
+ - ""
resources:
- - submarines/finalizers
- verbs:
- - update
-- apiGroups:
- - submarine.apache.org
- resources:
- - submarines/status
+ - pods
+ - pods/log
+ - services
+ - persistentvolumeclaims
+ - events
+ - configmaps
verbs:
- get
- - patch
- - update
+ - list
+ - watch
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: "submarine-observer"
+subjects:
+- kind: ServiceAccount
+ name: "default"
+roleRef:
+ kind: Role
+ name: "submarine-observer"
+ apiGroup: rbac.authorization.k8s.io
diff --git a/submarine-cloud-v3/artifacts/submarine-server-rbac.yaml
b/submarine-cloud-v3/artifacts/submarine-server-rbac.yaml
new file mode 100644
index 00000000..bf59bd3c
--- /dev/null
+++ b/submarine-cloud-v3/artifacts/submarine-server-rbac.yaml
@@ -0,0 +1,142 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: "submarine-server"
+rules:
+- apiGroups:
+ - kubeflow.org
+ resources:
+ - tfjobs
+ - tfjobs/status
+ - pytorchjobs
+ - pytorchjobs/status
+ - notebooks
+ - notebooks/status
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - delete
+ - deletecollection
+ - patch
+ - update
+- apiGroups:
+ - traefik.containo.us
+ resources:
+ - ingressroutes
+ - middlewares
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - delete
+ - deletecollection
+ - patch
+ - update
+- apiGroups:
+ - machinelearning.seldon.io
+ resources:
+ - seldondeployments
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - delete
+ - deletecollection
+ - patch
+ - update
+- apiGroups:
+ - networking.istio.io
+ resources:
+ - virtualservices
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - delete
+ - deletecollection
+ - patch
+ - update
+- apiGroups:
+ - ""
+ resources:
+ - pods
+ - pods/log
+ - services
+ - persistentvolumeclaims
+ - events
+ - configmaps
+ verbs:
+ - '*'
+- apiGroups:
+ - "apps"
+ resources:
+ - deployments
+ - deployments/status
+ verbs:
+ - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: "observer"
+rules:
+- apiGroups:
+ - kubeflow.org
+ resources:
+ - tfjobs
+ - tfjobs/status
+ - pytorchjobs
+ - pytorchjobs/status
+ - notebooks
+ - notebooks/status
+ verbs:
+ - get
+ - list
+ - watch
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: "submarine-server"
+subjects:
+- kind: ServiceAccount
+ name: "submarine-server"
+roleRef:
+ kind: Role
+ name: "submarine-server"
+ apiGroup: rbac.authorization.k8s.io
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: "observer"
+subjects:
+- kind: ServiceAccount
+ name: "default"
+roleRef:
+ kind: Role
+ name: "observer"
+ apiGroup: rbac.authorization.k8s.io
diff --git a/submarine-cloud-v3/artifacts/submarine-server.yaml
b/submarine-cloud-v3/artifacts/submarine-server.yaml
new file mode 100644
index 00000000..85d7b8b0
--- /dev/null
+++ b/submarine-cloud-v3/artifacts/submarine-server.yaml
@@ -0,0 +1,99 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: "submarine-server"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: "default"
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: "submarine-server"
+ labels:
+ app: "submarine-server"
+spec:
+ ports:
+ - port: 8080
+ targetPort: 8080
+ protocol: TCP
+ name: http
+ selector:
+ app: "submarine-server"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: "submarine-server"
+spec:
+ selector:
+ matchLabels:
+ app: "submarine-server"
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: "submarine-server"
+
+ spec:
+ serviceAccountName: "submarine-server"
+ initContainers:
+ - name: submarine-server-initcontainer
+ image: "minio/mc"
+ command: ["/bin/bash", "-c",
+ "cnt=0;
+ while ! /bin/bash -c 'mc --config-dir /root/.mc config host add minio
http://submarine-minio-service:9000
+ submarine_minio submarine_minio' 2>&1;
+ do
+ sleep 15;
+ ((cnt=cnt+1));
+ if [ $cnt -eq 80 ];then
+ echo 'ERROR: wait too long for minio pod';
+ exit 1;
+ fi;
+ done;
+ if /bin/bash -c 'mc --config-dir /root/.mc ls minio/submarine'
>/dev/null 2>&1; then
+ echo 'Bucket minio/submarine already exists, skipping creation.';
+ else
+ /bin/bash -c 'mc --config-dir /root/.mc mb minio/submarine';
+ fi;"]
+ volumeMounts:
+ - name: mc-config-vol
+ mountPath: /root/.mc
+ volumes:
+ - name: mc-config-vol
+ emptyDir: { }
+ containers:
+ - name: "submarine-server"
+ env:
+ - name: SUBMARINE_SERVER_PORT
+ value: "8080"
+ - name: SUBMARINE_SERVER_PORT_8080_TCP
+ value: "8080"
+ - name: K8S_APISERVER_URL
+ value: "kubernetes.default.svc"
+
+ image: "apache/submarine:server-0.8.0-SNAPSHOT"
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 8080
diff --git a/submarine-cloud-v3/config/rbac/role.yaml
b/submarine-cloud-v3/artifacts/submarine-storage-rbac.yaml
similarity index 66%
copy from submarine-cloud-v3/config/rbac/role.yaml
copy to submarine-cloud-v3/artifacts/submarine-storage-rbac.yaml
index e73b5772..5fedc387 100644
--- a/submarine-cloud-v3/config/rbac/role.yaml
+++ b/submarine-cloud-v3/artifacts/submarine-storage-rbac.yaml
@@ -17,34 +17,24 @@
---
apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
metadata:
- creationTimestamp: null
- name: manager-role
-rules:
-- apiGroups:
- - submarine.apache.org
- resources:
- - submarines
- verbs:
- - create
- - delete
- - get
- - list
- - patch
- - update
- - watch
-- apiGroups:
- - submarine.apache.org
- resources:
- - submarines/finalizers
- verbs:
- - update
-- apiGroups:
- - submarine.apache.org
- resources:
- - submarines/status
- verbs:
- - get
- - patch
- - update
+ name: "submarine-storage"
+rules: []
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: submarine-storage
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: "submarine-storage"
+subjects:
+ - kind: ServiceAccount
+ name: "submarine-storage"
+roleRef:
+ kind: Role
+ name: "submarine-storage"
+ apiGroup: rbac.authorization.k8s.io
diff --git a/submarine-cloud-v3/artifacts/submarine-tensorboard.yaml
b/submarine-cloud-v3/artifacts/submarine-tensorboard.yaml
new file mode 100644
index 00000000..b108f8bc
--- /dev/null
+++ b/submarine-cloud-v3/artifacts/submarine-tensorboard.yaml
@@ -0,0 +1,79 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: submarine-tensorboard-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ storageClassName: "submarine-storageclass"
+ resources:
+ requests:
+ storage: "10Gi"
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: submarine-tensorboard-service
+spec:
+ selector:
+ app: submarine-tensorboard
+ ports:
+ - protocol: TCP
+ port: 8080
+ targetPort: 6006
+ name: http
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: submarine-tensorboard
+spec:
+ selector:
+ matchLabels:
+ app: submarine-tensorboard
+ template:
+ metadata:
+ labels:
+ app: submarine-tensorboard
+ spec:
+ serviceAccountName: "submarine-storage"
+ containers:
+ - name: submarine-tensorboard-container
+ image: tensorflow/tensorflow:1.11.0
+ command:
+ - "tensorboard"
+ - "--logdir=/logs"
+ - "--path_prefix=/tensorboard"
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 6006
+ volumeMounts:
+ - mountPath: "/logs"
+ name: "volume"
+ subPath: "submarine-tensorboard"
+ readinessProbe:
+ tcpSocket:
+ port: 6006
+ periodSeconds: 10
+ volumes:
+ - name: "volume"
+ persistentVolumeClaim:
+ claimName: "submarine-tensorboard-pvc"
diff --git a/submarine-cloud-v3/artifacts/submarine-virtualservice.yaml
b/submarine-cloud-v3/artifacts/submarine-virtualservice.yaml
new file mode 100644
index 00000000..961ab006
--- /dev/null
+++ b/submarine-cloud-v3/artifacts/submarine-virtualservice.yaml
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+ name: submarine-virtual-service
+spec:
+ hosts:
+ - "*"
+ gateways:
+ - submarine/submarine-gateway
+ http:
+ - match:
+ - uri:
+ prefix: /tensorboard
+ route:
+ - destination:
+ host: submarine-tensorboard-service
+ port:
+ number: 8080
+ - match:
+ - uri:
+ prefix: /mlflow
+ route:
+ - destination:
+ host: submarine-mlflow-service
+ port:
+ number: 5000
+ - match:
+ - uri:
+ prefix: /minio
+ route:
+ - destination:
+ host: submarine-minio-service
+ port:
+ number: 9000
+ - route:
+ - destination:
+ host: submarine-server
+ port:
+ number: 8080
diff --git a/submarine-cloud-v3/config/rbac/role.yaml
b/submarine-cloud-v3/config/rbac/role.yaml
index e73b5772..7384f94b 100644
--- a/submarine-cloud-v3/config/rbac/role.yaml
+++ b/submarine-cloud-v3/config/rbac/role.yaml
@@ -22,6 +22,109 @@ metadata:
creationTimestamp: null
name: manager-role
rules:
+- apiGroups:
+ - ""
+ resources:
+ - events
+ verbs:
+ - create
+ - patch
+- apiGroups:
+ - apps
+ resources:
+ - deployments
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - apps
+ resources:
+ - statefulsets
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ - persistentvolumeclaims
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ - serviceaccounts
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ - services
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - networking.istio.io
+ resources:
+ - virtualservices
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - rbac
+ resources:
+ - rolebindings
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - rbac
+ resources:
+ - roles
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
- apiGroups:
- submarine.apache.org
resources:
diff --git a/submarine-cloud-v3/controllers/parser.go
b/submarine-cloud-v3/controllers/parser.go
new file mode 100644
index 00000000..d1231c69
--- /dev/null
+++ b/submarine-cloud-v3/controllers/parser.go
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+
+ "github.com/pkg/errors"
+ istiov1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
+ appsv1 "k8s.io/api/apps/v1"
+ v1 "k8s.io/api/core/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/util/yaml"
+)
+
+// PathToOSFile turn the file at the relativePath into a type of *os.File.
+func pathToOSFile(relativePath string) (*os.File, error) {
+ path, err := filepath.Abs(relativePath)
+ if err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("failed generate
absolute file path of %s", relativePath))
+ }
+
+ manifest, err := os.Open(path)
+ if err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("failed to open file
%s", path))
+ }
+
+ return manifest, nil
+}
+
+// ParseYaml
+func parseYaml(relativePath, kind string) ([]byte, error) {
+ var manifest *os.File
+ var err error
+
+ var marshaled []byte
+ if manifest, err = pathToOSFile(relativePath); err != nil {
+ return nil, err
+ }
+
+ decoder := yaml.NewYAMLOrJSONDecoder(manifest, 100)
+ for {
+ var out unstructured.Unstructured
+ err = decoder.Decode(&out)
+ if err != nil {
+ // this would indicate it's malformed YAML.
+ break
+ }
+
+ if out.GetKind() == kind {
+ marshaled, err = out.MarshalJSON()
+ break
+ }
+ }
+
+ if err != io.EOF && err != nil {
+ return nil, err
+ }
+ return marshaled, nil
+}
+
+// ParseServiceAccount parse ServiceAccount from yaml file.
+func ParseServiceAccountYaml(relativePath string) (*v1.ServiceAccount, error) {
+ var serviceAccount v1.ServiceAccount
+ marshaled, err := parseYaml(relativePath, "ServiceAccount")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &serviceAccount)
+ return &serviceAccount, nil
+}
+
+// ParseDeploymentYaml parse Deployment from yaml file.
+func ParseDeploymentYaml(relativePath string) (*appsv1.Deployment, error) {
+ var deployment appsv1.Deployment
+ marshaled, err := parseYaml(relativePath, "Deployment")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &deployment)
+ return &deployment, nil
+}
+
+// ParseStatefulSetYaml parse StatefulSets from yaml file.
+func ParseStatefulSetYaml(relativePath string) (*appsv1.StatefulSet, error) {
+ var statefulset appsv1.StatefulSet
+ marshaled, err := parseYaml(relativePath, "StatefulSet")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &statefulset)
+ return &statefulset, nil
+}
+
+// ParseServiceYaml parse Service from yaml file.
+func ParseServiceYaml(relativePath string) (*v1.Service, error) {
+ var service v1.Service
+ marshaled, err := parseYaml(relativePath, "Service")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &service)
+ return &service, nil
+}
+
+// ParseRoleBindingYaml parse RoleBinding from yaml file.
+func ParseRoleBindingYaml(relativePath string) (*rbacv1.RoleBinding, error) {
+ var rolebinding rbacv1.RoleBinding
+ marshaled, err := parseYaml(relativePath, "RoleBinding")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &rolebinding)
+ return &rolebinding, nil
+}
+
+// ParseRoleYaml parse Role from yaml file.
+func ParseRoleYaml(relativePath string) (*rbacv1.Role, error) {
+ var role rbacv1.Role
+ marshaled, err := parseYaml(relativePath, "Role")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &role)
+ return &role, nil
+}
+
+// ParsePersistentVolumeClaimYaml parse PersistentVolumeClaimYaml from yaml
file.
+func ParsePersistentVolumeClaimYaml(relativePath string)
(*v1.PersistentVolumeClaim, error) {
+ var pvc v1.PersistentVolumeClaim
+ marshaled, err := parseYaml(relativePath, "PersistentVolumeClaim")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &pvc)
+ return &pvc, nil
+}
+
+// ParseVirtualService parse VirtualService from yaml file.
+func ParseVirtualService(relativePath string) (*istiov1alpha3.VirtualService,
error) {
+ var virtualService istiov1alpha3.VirtualService
+ marshaled, err := parseYaml(relativePath, "VirtualService")
+ if err != nil {
+ return nil, err
+ }
+ json.Unmarshal(marshaled, &virtualService)
+ return &virtualService, nil
+}
diff --git a/submarine-cloud-v3/controllers/submarine_controller.go
b/submarine-cloud-v3/controllers/submarine_controller.go
index 58d7b81e..fa334c8f 100644
--- a/submarine-cloud-v3/controllers/submarine_controller.go
+++ b/submarine-cloud-v3/controllers/submarine_controller.go
@@ -18,42 +18,401 @@ package controllers
import (
"context"
+ "encoding/json"
+ "fmt"
+ "github.com/go-logr/logr"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ "k8s.io/apimachinery/pkg/api/equality"
+ "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
- "sigs.k8s.io/controller-runtime/pkg/log"
submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
)
+// Defines resource names and path to artifact yaml files
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/
+const (
+ serverName = "submarine-server"
+ observerName = "submarine-observer"
+ databaseName = "submarine-database"
+ tensorboardName = "submarine-tensorboard"
+ mlflowName = "submarine-mlflow"
+ minioName = "submarine-minio"
+ storageName = "submarine-storage"
+ virtualServiceName = "submarine-virtual-service"
+ databasePvcName = databaseName + "-pvc"
+ tensorboardPvcName = tensorboardName + "-pvc"
+ tensorboardServiceName = tensorboardName + "-service"
+ mlflowPvcName = mlflowName + "-pvc"
+ mlflowServiceName = mlflowName + "-service"
+ minioPvcName = minioName + "-pvc"
+ minioServiceName = minioName + "-service"
+ artifactPath = "./artifacts/"
+ databaseYamlPath = artifactPath + "submarine-database.yaml"
+ minioYamlPath = artifactPath + "submarine-minio.yaml"
+ mlflowYamlPath = artifactPath + "submarine-mlflow.yaml"
+ serverYamlPath = artifactPath + "submarine-server.yaml"
+ tensorboardYamlPath = artifactPath + "submarine-tensorboard.yaml"
+ serverRbacYamlPath = artifactPath + "submarine-server-rbac.yaml"
+ observerRbacYamlPath = artifactPath + "submarine-observer-rbac.yaml"
+ storageRbacYamlPath = artifactPath + "submarine-storage-rbac.yaml"
+ virtualServiceYamlPath = artifactPath + "submarine-virtualservice.yaml"
+)
+
+// Name of deployments whose replica count and readiness need to be checked
+var dependents = []string{serverName, tensorboardName, mlflowName, minioName}
+
+const (
+ // SuccessSynced is used as part of the Event 'reason' when a Submarine
is synced
+ //SuccessSynced = "Synced"
+
+ // MessageResourceSynced is the message used for an Event fired when a
+ // Submarine is synced successfully
+ //MessageResourceSynced = "Submarine synced successfully"
+
+ // ErrResourceExists is used as part of the Event 'reason' when a
Submarine fails
+ // to sync due to a Deployment of the same name already existing.
+ ErrResourceExists = "ErrResourceExists"
+
+ // MessageResourceExists is the message used for Events when a resource
+ // fails to sync due to a Deployment already existing
+ MessageResourceExists = "Resource %q already exists and is not managed
by Submarine"
+)
+
+// Default k8s anyuid role rule
+var k8sAnyuidRoleRule = rbacv1.PolicyRule{
+ APIGroups: []string{"policy"},
+ Verbs: []string{"use"},
+ Resources: []string{"podsecuritypolicies"},
+ ResourceNames: []string{"submarine-anyuid"},
+}
+
+// Openshift anyuid role rule
+var openshiftAnyuidRoleRule = rbacv1.PolicyRule{
+ APIGroups: []string{"security.openshift.io"},
+ Verbs: []string{"use"},
+ Resources: []string{"securitycontextconstraints"},
+ ResourceNames: []string{"anyuid"},
+}
+
// SubmarineReconciler reconciles a Submarine object
type SubmarineReconciler struct {
+ // Fields required by the operator
client.Client
- Scheme *runtime.Scheme
+ Scheme *runtime.Scheme
+ Log logr.Logger
+ Recorder record.EventRecorder
+ // Fields required by submarine
+ ClusterType string
+ CreatePodSecurityPolicy bool
}
+// kubebuilder RBAC markers generates rules for the operator ClusterRole
+// On change, run `make manifest` to update config/rbac/role.yaml
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/config/rbac/role.yaml
+
+// Submarine resource
//+kubebuilder:rbac:groups=submarine.apache.org,resources=submarines,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=submarine.apache.org,resources=submarines/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=submarine.apache.org,resources=submarines/finalizers,verbs=update
+// Event
+//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch
+
+// Other resources
+//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=rbac,resources=roles,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=rbac,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=networking.istio.io,resources=virtualservices,verbs=get;list;watch;create;update;patch;delete
+
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
-// TODO(user): Modify the Reconcile function to compare the state specified by
-// the Submarine object against the actual cluster state, and then
-// perform operations to make the cluster state reflect the state specified by
-// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
func (r *SubmarineReconciler) Reconcile(ctx context.Context, req ctrl.Request)
(ctrl.Result, error) {
- _ = log.FromContext(ctx)
+ r.Log.Info("Enter Reconcile", "req", req)
+
+ // Get the Submarine resource with the requested name/namespace
+ submarine := &submarineapacheorgv1alpha1.Submarine{}
+ err := r.Get(ctx, types.NamespacedName{Name: req.Name, Namespace:
req.Namespace}, submarine)
+ if err != nil {
+ if errors.IsNotFound(err) {
+ // The Submarine resource may no longer exist, in which
case we stop processing
+ r.Log.Error(nil, "Submarine no longer exists", "name",
req.Name, "namespace", req.Namespace)
+ return ctrl.Result{}, nil
+ }
+ return ctrl.Result{}, err
+ }
+
+ // Submarine is in the terminating process, only used when in
foreground cascading deletion, otherwise the submarine will be recreated
+ if !submarine.DeletionTimestamp.IsZero() {
+ return ctrl.Result{}, nil
+ }
- // TODO(user): your logic here
+ submarineCopy := submarine.DeepCopy()
+
+ // Take action based on submarine state
+ // State machine for Submarine:
+ //+-----------------------------------------------------------------+
+ //| +---------+ +----------+ +----------+ |
+ //| | | | | | | |
+ //| | New +---------> Creating +----------> Running | |
+ //| | | | | | | |
+ //| +----+----+ +-----+----+ +-----+----+ |
+ //| | | | |
+ //| | | | |
+ //| | | | |
+ //| | | +-----v----+ |
+ //| | | | | |
+ //| +--------------------+---------------> Failed | |
+ //| | | |
+ //| +----------+ |
+ //+-----------------------------------------------------------------+
+ switch submarineCopy.Status.State {
+ case submarineapacheorgv1alpha1.NewState:
+ r.recordSubmarineEvent(submarineCopy)
+ if err := r.validateSubmarine(submarineCopy); err != nil {
+ submarineCopy.Status.State =
submarineapacheorgv1alpha1.FailedState
+ submarineCopy.Status.ErrorMessage = err.Error()
+ r.recordSubmarineEvent(submarineCopy)
+ } else {
+ submarineCopy.Status.State =
submarineapacheorgv1alpha1.CreatingState
+ r.recordSubmarineEvent(submarineCopy)
+ }
+ case submarineapacheorgv1alpha1.CreatingState:
+ if err := r.createSubmarine(ctx, submarineCopy); err != nil {
+ submarineCopy.Status.State =
submarineapacheorgv1alpha1.FailedState
+ submarineCopy.Status.ErrorMessage = err.Error()
+ r.recordSubmarineEvent(submarineCopy)
+ }
+ ok, err := r.checkSubmarineDependentsReady(ctx, submarineCopy)
+ if err != nil {
+ submarineCopy.Status.State =
submarineapacheorgv1alpha1.FailedState
+ submarineCopy.Status.ErrorMessage = err.Error()
+ r.recordSubmarineEvent(submarineCopy)
+ }
+ if ok {
+ submarineCopy.Status.State =
submarineapacheorgv1alpha1.RunningState
+ r.recordSubmarineEvent(submarineCopy)
+ }
+ case submarineapacheorgv1alpha1.RunningState:
+ if err := r.createSubmarine(ctx, submarineCopy); err != nil {
+ submarineCopy.Status.State =
submarineapacheorgv1alpha1.FailedState
+ submarineCopy.Status.ErrorMessage = err.Error()
+ r.recordSubmarineEvent(submarineCopy)
+ }
+ }
+
+ // Update STATUS of Submarine
+ err = r.updateSubmarineStatus(ctx, submarine, submarineCopy)
+ if err != nil {
+ return ctrl.Result{}, err
+ }
return ctrl.Result{}, nil
}
+func (r *SubmarineReconciler) updateSubmarineStatus(ctx context.Context,
submarine, submarineCopy *submarineapacheorgv1alpha1.Submarine) error {
+ // Update server replicas
+ serverDeployment := &appsv1.Deployment{}
+ err := r.Get(ctx, types.NamespacedName{Name: serverName, Namespace:
submarine.Namespace}, serverDeployment)
+ if err != nil {
+ if errors.IsNotFound(err) {
+ submarineCopy.Status.AvailableServerReplicas =
serverDeployment.Status.AvailableReplicas
+ } else {
+ return err
+ }
+ }
+
+ // Update database replicas
+ statefulset := &appsv1.StatefulSet{}
+ err = r.Get(ctx, types.NamespacedName{Name: databaseName, Namespace:
submarine.Namespace}, statefulset)
+ if err != nil {
+ if errors.IsNotFound(err) {
+ submarineCopy.Status.AvailableDatabaseReplicas =
statefulset.Status.ReadyReplicas
+ } else {
+ return err
+ }
+ }
+
+ // Skip update if nothing changed.
+ if equality.Semantic.DeepEqual(submarine.Status, submarineCopy.Status) {
+ return nil
+ }
+
+ // Update submarine status
+ err = r.Status().Update(ctx, submarineCopy)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (r *SubmarineReconciler) validateSubmarine(submarine
*submarineapacheorgv1alpha1.Submarine) error {
+ // Print out the spec of the Submarine resource
+ b, err := json.MarshalIndent(submarine.Spec, "", " ")
+ fmt.Println(string(b))
+
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// Creates resources according to artifact yaml files
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/
+func (r *SubmarineReconciler) createSubmarine(ctx context.Context, submarine
*submarineapacheorgv1alpha1.Submarine) error {
+ var err error
+ // We create rbac first, this ensures that any dependency based on it
will not go wrong
+ err = r.createSubmarineServerRBAC(ctx, submarine)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ err = r.createSubmarineStorageRBAC(ctx, submarine)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ err = r.createSubmarineObserverRBAC(ctx, submarine)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ err = r.createSubmarineServer(ctx, submarine)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ err = r.createSubmarineDatabase(ctx, submarine)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ err = r.createVirtualService(ctx, submarine)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ err = r.createSubmarineTensorboard(ctx, submarine)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ err = r.createSubmarineMlflow(ctx, submarine)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ err = r.createSubmarineMinio(ctx, submarine)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ return nil
+}
+
+// Checks the number of deployment and database replicas and if they are ready
+func (r *SubmarineReconciler) checkSubmarineDependentsReady(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine) (bool, error)
{
+ // deployment dependents check
+ for _, name := range dependents {
+ // 1. Check if deployment exists
+ deployment := &appsv1.Deployment{}
+ err := r.Get(ctx, types.NamespacedName{Name: name, Namespace:
submarine.Namespace}, deployment)
+ if err != nil {
+ if errors.IsNotFound(err) {
+ return false, nil
+ }
+ return false, err
+ }
+ // 2. Check if deployment replicas failed
+ for _, condition := range deployment.Status.Conditions {
+ if condition.Type == appsv1.DeploymentReplicaFailure {
+ return false, fmt.Errorf("failed creating
replicas of %s, message: %s", deployment.Name, condition.Message)
+ }
+ }
+ // 3. Check if ready replicas are same as targeted replicas
+ if deployment.Status.ReadyReplicas !=
deployment.Status.Replicas {
+ return false, nil
+ }
+ }
+ // database check
+ // 1. Check if database exists
+ statefulset := &appsv1.StatefulSet{}
+ err := r.Get(ctx, types.NamespacedName{Name: databaseName, Namespace:
submarine.Namespace}, statefulset)
+ if err != nil {
+ if errors.IsNotFound(err) {
+ return false, nil
+ }
+ return false, err
+ }
+
+ // 2. Check if database replicas failed
+ // statefulset.Status.Conditions does not have the specified type enum
like
+ // deployment.Status.Conditions => DeploymentConditionType ,
+ // so we will ignore the verification status for the time being
+
+ // 3. Check if ready replicas are same as targeted replicas
+ if statefulset.Status.Replicas != statefulset.Status.ReadyReplicas {
+ return false, nil
+ }
+
+ return true, nil
+}
+
+// Wraps r.Recorder.Eventf
+// Fill reason, message fields of Event according to state of Submarine
+func (r *SubmarineReconciler) recordSubmarineEvent(submarine
*submarineapacheorgv1alpha1.Submarine) {
+ switch submarine.Status.State {
+ case submarineapacheorgv1alpha1.NewState:
+ r.Recorder.Eventf(
+ submarine,
+ corev1.EventTypeNormal,
+ "SubmarineAdded",
+ "Submarine %s was added",
+ submarine.Name)
+ case submarineapacheorgv1alpha1.CreatingState:
+ r.Recorder.Eventf(
+ submarine,
+ corev1.EventTypeNormal,
+ "SubmarineCreating",
+ "Submarine %s was creating",
+ submarine.Name,
+ )
+ case submarineapacheorgv1alpha1.RunningState:
+ r.Recorder.Eventf(
+ submarine,
+ corev1.EventTypeNormal,
+ "SubmarineRunning",
+ "Submarine %s was running",
+ submarine.Name,
+ )
+ case submarineapacheorgv1alpha1.FailedState:
+ r.Recorder.Eventf(
+ submarine,
+ corev1.EventTypeWarning,
+ "SubmarineFailed",
+ "Submarine %s was failed: %s",
+ submarine.Name,
+ submarine.Status.SubmarineState.ErrorMessage,
+ )
+ }
+}
+
// SetupWithManager sets up the controller with the Manager.
func (r *SubmarineReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
diff --git a/submarine-cloud-v3/controllers/submarine_database.go
b/submarine-cloud-v3/controllers/submarine_database.go
new file mode 100644
index 00000000..91e48aa8
--- /dev/null
+++ b/submarine-cloud-v3/controllers/submarine_database.go
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+
+ submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
+
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *SubmarineReconciler) newSubmarineDatabasePersistentVolumeClaim(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*corev1.PersistentVolumeClaim {
+ pvc, err := ParsePersistentVolumeClaimYaml(databaseYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParsePersistentVolumeClaimYaml")
+ }
+ pvc.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, pvc, r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set PVC ControllerReference")
+ }
+ return pvc
+}
+
+func (r *SubmarineReconciler) newSubmarineDatabaseStatefulSet(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*appsv1.StatefulSet {
+ statefulset, err := ParseStatefulSetYaml(databaseYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseStatefulSetYaml")
+ }
+
+ statefulset.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, statefulset,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Stateful Set ControllerReference")
+ }
+
+ databaseImage := submarine.Spec.Database.Image
+ if databaseImage != "" {
+ statefulset.Spec.Template.Spec.Containers[0].Image =
databaseImage
+ }
+
+ return statefulset
+}
+
+func (r *SubmarineReconciler) newSubmarineDatabaseService(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) *corev1.Service {
+ service, err := ParseServiceYaml(databaseYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseServiceYaml")
+ }
+ service.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, service,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Service ControllerReference")
+ }
+ return service
+}
+
+// createSubmarineDatabase is a function to create submarine-database.
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/submarine-database.yaml
+func (r *SubmarineReconciler) createSubmarineDatabase(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) error {
+ r.Log.Info("Enter createSubmarineDatabase")
+
+ // Step 1: Create PersistentVolumeClaim
+ pvc := &corev1.PersistentVolumeClaim{}
+ err := r.Get(ctx, types.NamespacedName{Name: databasePvcName,
Namespace: submarine.Namespace}, pvc)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ pvc = r.newSubmarineDatabasePersistentVolumeClaim(ctx,
submarine)
+ err = r.Create(ctx, pvc)
+ r.Log.Info("Create PersistentVolumeClaim", "name", pvc.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(pvc, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, pvc.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step 2: Create Statefulset
+ statefulset := &appsv1.StatefulSet{}
+ err = r.Get(ctx, types.NamespacedName{Name: databaseName, Namespace:
submarine.Namespace}, statefulset)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ statefulset = r.newSubmarineDatabaseStatefulSet(ctx, submarine)
+ err = r.Create(ctx, statefulset)
+ r.Log.Info("Create StatefulSet", "name", statefulset.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(statefulset, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, statefulset.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ if err != nil {
+ return err
+ }
+
+ // Step 3: Create Service
+ service := &corev1.Service{}
+ err = r.Get(ctx, types.NamespacedName{Name: databaseName, Namespace:
submarine.Namespace}, service)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ service = r.newSubmarineDatabaseService(ctx, submarine)
+ err = r.Create(ctx, service)
+ r.Log.Info("Create Service", "name", service.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(service, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, service.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ return nil
+}
diff --git a/submarine-cloud-v3/controllers/submarine_minio.go
b/submarine-cloud-v3/controllers/submarine_minio.go
new file mode 100644
index 00000000..b5d5e8d6
--- /dev/null
+++ b/submarine-cloud-v3/controllers/submarine_minio.go
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+
+ submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
+
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *SubmarineReconciler) newSubmarineMinioPersistentVolumeClaim(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*corev1.PersistentVolumeClaim {
+ pvc, err := ParsePersistentVolumeClaimYaml(minioYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParsePersistentVolumeClaimYaml")
+ }
+ pvc.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, pvc, r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set PersistentVolumeClaim
ControllerReference")
+ }
+ return pvc
+}
+
+func (r *SubmarineReconciler) newSubmarineMinioDeployment(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) *appsv1.Deployment {
+ deployment, err := ParseDeploymentYaml(minioYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseDeploymentYaml")
+ }
+ deployment.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, deployment,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Deployment ControllerReference")
+ }
+ return deployment
+}
+
+func (r *SubmarineReconciler) newSubmarineMinioService(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) *corev1.Service {
+ service, err := ParseServiceYaml(minioYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseServiceYaml")
+ }
+ service.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, service,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Service ControllerReference")
+ }
+ return service
+}
+
+// createSubmarineMinio is a function to create submarine-minio.
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/submarine-minio.yaml
+func (r *SubmarineReconciler) createSubmarineMinio(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) error {
+ r.Log.Info("Enter createSubmarineMinio")
+
+ // Step 1: Create PersistentVolumeClaim
+ pvc := &corev1.PersistentVolumeClaim{}
+ err := r.Get(ctx, types.NamespacedName{Name: minioPvcName, Namespace:
submarine.Namespace}, pvc)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ pvc = r.newSubmarineMinioPersistentVolumeClaim(ctx, submarine)
+ err = r.Create(ctx, pvc)
+ r.Log.Info("Create PersistentVolumeClaim", "name", pvc.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(pvc, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, pvc.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step 2: Create Deployment
+ deployment := &appsv1.Deployment{}
+ err = r.Get(ctx, types.NamespacedName{Name: minioName, Namespace:
submarine.Namespace}, deployment)
+ if errors.IsNotFound(err) {
+ deployment = r.newSubmarineMinioDeployment(ctx, submarine)
+ err = r.Create(ctx, deployment)
+ r.Log.Info("Create Deployment", "name", deployment.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(deployment, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, deployment.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step 3: Create Service
+ service := &corev1.Service{}
+ err = r.Get(ctx, types.NamespacedName{Name: minioServiceName,
Namespace: submarine.Namespace}, service)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ service = r.newSubmarineMinioService(ctx, submarine)
+ err = r.Create(ctx, service)
+ r.Log.Info("Create Service", "name", service.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(service, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, service.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ return nil
+}
diff --git a/submarine-cloud-v3/controllers/submarine_mlflow.go
b/submarine-cloud-v3/controllers/submarine_mlflow.go
new file mode 100644
index 00000000..5a345776
--- /dev/null
+++ b/submarine-cloud-v3/controllers/submarine_mlflow.go
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+
+ submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
+
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *SubmarineReconciler) newSubmarineMlflowPersistentVolumeClaim(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*corev1.PersistentVolumeClaim {
+ pvc, err := ParsePersistentVolumeClaimYaml(mlflowYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParsePersistentVolumeClaimYaml")
+ }
+ pvc.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, pvc, r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set PersistentVolumeClaim
ControllerReference")
+ }
+ return pvc
+}
+
+func (r *SubmarineReconciler) newSubmarineMlflowDeployment(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*appsv1.Deployment {
+ deployment, err := ParseDeploymentYaml(mlflowYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseDeploymentYaml")
+ }
+ deployment.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, deployment,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Deployment ControllerReference")
+ }
+ return deployment
+}
+
+func (r *SubmarineReconciler) newSubmarineMlflowService(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) *corev1.Service {
+ service, err := ParseServiceYaml(mlflowYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseServiceYaml")
+ }
+ service.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, service,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Service ControllerReference")
+ }
+ return service
+}
+
+// createSubmarineMlflow is a function to create submarine-mlflow.
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/submarine-mlflow.yaml
+func (r *SubmarineReconciler) createSubmarineMlflow(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) error {
+ r.Log.Info("Enter createSubmarineMlflow")
+
+ // Step 1: Create PersistentVolumeClaim
+ pvc := &corev1.PersistentVolumeClaim{}
+ err := r.Get(ctx, types.NamespacedName{Name: mlflowPvcName, Namespace:
submarine.Namespace}, pvc)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ pvc = r.newSubmarineMlflowPersistentVolumeClaim(ctx, submarine)
+ err = r.Create(ctx, pvc)
+ r.Log.Info("Create PersistentVolumeClaim", "name", pvc.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(pvc, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, pvc.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step 2: Create Deployment
+ deployment := &appsv1.Deployment{}
+ err = r.Get(ctx, types.NamespacedName{Name: mlflowName, Namespace:
submarine.Namespace}, deployment)
+ if errors.IsNotFound(err) {
+ deployment = r.newSubmarineMlflowDeployment(ctx, submarine)
+ err = r.Create(ctx, deployment)
+ r.Log.Info("Create Deployment", "name", deployment.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(deployment, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, deployment.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step 3: Create Service
+ service := &corev1.Service{}
+ err = r.Get(ctx, types.NamespacedName{Name: mlflowServiceName,
Namespace: submarine.Namespace}, service)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ // If an error occurs during Get/Create, we'll requeue the item
so we can
+ // attempt processing again later. This could have been caused
by a
+ // temporary network failure, or any other transient reason.
+ service = r.newSubmarineMlflowService(ctx, submarine)
+ err = r.Create(ctx, service)
+ r.Log.Info("Create Service", "name", service.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(service, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, service.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ return nil
+}
diff --git a/submarine-cloud-v3/controllers/submarine_observer_rbac.go
b/submarine-cloud-v3/controllers/submarine_observer_rbac.go
new file mode 100644
index 00000000..b1126b7b
--- /dev/null
+++ b/submarine-cloud-v3/controllers/submarine_observer_rbac.go
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+
+ corev1 "k8s.io/api/core/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+
+ submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
+
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *SubmarineReconciler) newSubmarineObserverRole(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) *rbacv1.Role {
+ role, err := ParseRoleYaml(observerRbacYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseRoleYaml")
+ }
+ role.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, role, r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Role ControllerReference")
+ }
+ return role
+}
+
+func (r *SubmarineReconciler) newSubmarineObserverRoleBinding(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*rbacv1.RoleBinding {
+ roleBinding, err := ParseRoleBindingYaml(observerRbacYamlPath)
+ if err != nil {
+ r.Log.Error(err, "Set RoleBinding ControllerReference")
+ }
+ roleBinding.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, roleBinding,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set RoleBinding ControllerReference")
+ }
+ return roleBinding
+}
+
+// createSubmarineObserverRBAC is a function to create RBAC for
submarine-observer which will be binded on service account: default.
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/submarine-observer-rbac.yaml
+func (r *SubmarineReconciler) createSubmarineObserverRBAC(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) error {
+ r.Log.Info("Enter createSubmarineObserverRBAC")
+
+ // Step1: Create Role
+ role := &rbacv1.Role{}
+ err := r.Get(ctx, types.NamespacedName{Name: observerName, Namespace:
submarine.Namespace}, role)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ role = r.newSubmarineObserverRole(ctx, submarine)
+ err = r.Create(ctx, role)
+ r.Log.Info("Create Role", "name", role.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(role, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, role.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step2: Create Role Binding
+ rolebinding := &rbacv1.RoleBinding{}
+ err = r.Get(ctx, types.NamespacedName{Name: observerName, Namespace:
submarine.Namespace}, rolebinding)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ rolebinding = r.newSubmarineObserverRoleBinding(ctx, submarine)
+ err = r.Create(ctx, rolebinding)
+ r.Log.Info("Create RoleBinding", "name", rolebinding.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(rolebinding, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, rolebinding.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ return nil
+}
diff --git a/submarine-cloud-v3/controllers/submarine_server.go
b/submarine-cloud-v3/controllers/submarine_server.go
new file mode 100644
index 00000000..f02e13ec
--- /dev/null
+++ b/submarine-cloud-v3/controllers/submarine_server.go
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+
+ submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
+
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *SubmarineReconciler) newSubmarineServerServiceAccount(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*corev1.ServiceAccount {
+ serviceAccount, err := ParseServiceAccountYaml(serverYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseServiceAccountYaml")
+ }
+ serviceAccount.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, serviceAccount,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set ServiceAccount ControllerReference")
+ }
+ return serviceAccount
+}
+
+func (r *SubmarineReconciler) newSubmarineServerService(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) *corev1.Service {
+ service, err := ParseServiceYaml(serverYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseServiceYaml")
+ }
+ service.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, service,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Service ControllerReference")
+ }
+ return service
+}
+
+func (r *SubmarineReconciler) newSubmarineServerDeployment(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*appsv1.Deployment {
+ serverReplicas := *submarine.Spec.Server.Replicas
+ operatorEnv := []corev1.EnvVar{
+ {
+ Name: "SUBMARINE_SERVER_DNS_NAME",
+ Value: serverName + "." + submarine.Namespace,
+ },
+ {
+ Name: "ENV_NAMESPACE",
+ Value: submarine.Namespace,
+ },
+ {
+ Name: "SUBMARINE_APIVERSION",
+ Value: submarine.APIVersion,
+ },
+ {
+ Name: "SUBMARINE_KIND",
+ Value: submarine.Kind,
+ },
+ {
+ Name: "SUBMARINE_NAME",
+ Value: submarine.Name,
+ },
+ {
+ Name: "SUBMARINE_UID",
+ Value: string(submarine.UID),
+ },
+ }
+
+ deployment, err := ParseDeploymentYaml(serverYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseDeploymentYaml")
+ }
+ deployment.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, deployment,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Deployment ControllerReference")
+ }
+ deployment.Spec.Replicas = &serverReplicas
+ deployment.Spec.Template.Spec.Containers[0].Env =
append(deployment.Spec.Template.Spec.Containers[0].Env, operatorEnv...)
+
+ return deployment
+}
+
+// createSubmarineServer is a function to create submarine-server.
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/submarine-server.yaml
+func (r *SubmarineReconciler) createSubmarineServer(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) error {
+ r.Log.Info("Enter createSubmarineServer")
+
+ // Step1: Create ServiceAccount
+ serviceaccount := &corev1.ServiceAccount{}
+ err := r.Get(ctx, types.NamespacedName{Name: serverName, Namespace:
submarine.Namespace}, serviceaccount)
+
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ serviceaccount = r.newSubmarineServerServiceAccount(ctx,
submarine)
+ err = r.Create(ctx, serviceaccount)
+ r.Log.Info("Create ServiceAccount", "name", serviceaccount.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(serviceaccount, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, serviceaccount.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step2: Create Service
+ service := &corev1.Service{}
+ err = r.Get(ctx, types.NamespacedName{Name: serverName, Namespace:
submarine.Namespace}, service)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ service = r.newSubmarineServerService(ctx, submarine)
+ err = r.Create(ctx, service)
+ r.Log.Info("Create Service", "name", service.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ if errors.IsNotFound(err) {
+ return nil
+ }
+ return err
+ }
+
+ if !metav1.IsControlledBy(service, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, service.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step3: Create Deployment
+ deployment := &appsv1.Deployment{}
+ err = r.Get(ctx, types.NamespacedName{Name: serverName, Namespace:
submarine.Namespace}, deployment)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ deployment = r.newSubmarineServerDeployment(ctx, submarine)
+ err = r.Create(ctx, deployment)
+ r.Log.Info("Create Deployment", "name", deployment.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ if errors.IsNotFound(err) {
+ return nil
+ }
+ return err
+ }
+
+ if !metav1.IsControlledBy(deployment, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, deployment.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Update the replicas of the server deployment if it is not equal to
spec
+ if submarine.Spec.Server.Replicas != nil &&
*submarine.Spec.Server.Replicas != *deployment.Spec.Replicas {
+ msg := fmt.Sprintf("Submarine %s server spec replicas",
submarine.Name)
+ r.Log.Info(msg, "server spec", *submarine.Spec.Server.Replicas,
"actual", *deployment.Spec.Replicas)
+
+ deployment = r.newSubmarineServerDeployment(ctx, submarine)
+ err = r.Update(ctx, deployment)
+ }
+
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/submarine-cloud-v3/controllers/submarine_server_rbac.go
b/submarine-cloud-v3/controllers/submarine_server_rbac.go
new file mode 100644
index 00000000..65336121
--- /dev/null
+++ b/submarine-cloud-v3/controllers/submarine_server_rbac.go
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+
+ corev1 "k8s.io/api/core/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+
+ submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
+
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *SubmarineReconciler) newSubmarineServerRole(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) *rbacv1.Role {
+ role, err := ParseRoleYaml(serverRbacYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseRoleYaml")
+ }
+ role.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, role, r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Role ControllerReference")
+ }
+
+ if r.CreatePodSecurityPolicy {
+ // If cluster type is openshift and need create pod security
policy, we need add anyuid scc, or we add k8s psp
+ if r.ClusterType == "openshift" {
+ role.Rules = append(role.Rules, openshiftAnyuidRoleRule)
+ } else {
+ role.Rules = append(role.Rules, k8sAnyuidRoleRule)
+ }
+ }
+
+ return role
+}
+
+func (r *SubmarineReconciler) newSubmarineServerRoleBinding(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*rbacv1.RoleBinding {
+ roleBinding, err := ParseRoleBindingYaml(serverRbacYamlPath)
+ if err != nil {
+ r.Log.Error(err, "Set RoleBinding ControllerReference")
+ }
+ roleBinding.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, roleBinding,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set RoleBinding ControllerReference")
+ }
+ return roleBinding
+}
+
+// createSubmarineServerRBAC is a function to create RBAC for submarine-server.
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/submarine-server-rbac.yaml
+func (r *SubmarineReconciler) createSubmarineServerRBAC(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) error {
+ r.Log.Info("Enter createSubmarineServerRBAC")
+
+ // Step1: Create Role
+ role := &rbacv1.Role{}
+ err := r.Get(ctx, types.NamespacedName{Name: serverName, Namespace:
submarine.Namespace}, role)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ role = r.newSubmarineServerRole(ctx, submarine)
+ err = r.Create(ctx, role)
+ r.Log.Info("Create Role", "name", role.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(role, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, role.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step2: Create Role Binding
+ rolebinding := &rbacv1.RoleBinding{}
+ err = r.Get(ctx, types.NamespacedName{Name: serverName, Namespace:
submarine.Namespace}, rolebinding)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ rolebinding = r.newSubmarineServerRoleBinding(ctx, submarine)
+ err = r.Create(ctx, rolebinding)
+ r.Log.Info("Create RoleBinding", "name", rolebinding.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(rolebinding, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, rolebinding.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ return nil
+}
diff --git a/submarine-cloud-v3/controllers/submarine_storage_rbac.go
b/submarine-cloud-v3/controllers/submarine_storage_rbac.go
new file mode 100644
index 00000000..7a6d2234
--- /dev/null
+++ b/submarine-cloud-v3/controllers/submarine_storage_rbac.go
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+
+ corev1 "k8s.io/api/core/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+
+ submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
+
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *SubmarineReconciler) newSubmarineStorageRole(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) *rbacv1.Role {
+ role, err := ParseRoleYaml(storageRbacYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseRoleYaml")
+ }
+ role.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, role, r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Role ControllerReference")
+ }
+
+ // If cluster type is openshift and need create pod security policy, we
need add anyuid scc, or we add k8s psp
+ if r.ClusterType == "openshift" {
+ role.Rules = append(role.Rules, openshiftAnyuidRoleRule)
+ } else {
+ role.Rules = append(role.Rules, k8sAnyuidRoleRule)
+ }
+
+ return role
+}
+
+func (r *SubmarineReconciler) newSubmarineStorageRoleBinding(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*rbacv1.RoleBinding {
+ roleBinding, err := ParseRoleBindingYaml(storageRbacYamlPath)
+ if err != nil {
+ r.Log.Error(err, "Set RoleBinding ControllerReference")
+ }
+ roleBinding.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, roleBinding,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set RoleBinding ControllerReference")
+ }
+ return roleBinding
+}
+
+func (r *SubmarineReconciler) newSubmarineStorageServiceAccount(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*corev1.ServiceAccount {
+ serviceAccount, err := ParseServiceAccountYaml(storageRbacYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseServiceAccountYaml")
+ }
+ serviceAccount.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, serviceAccount,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set ServiceAccount ControllerReference")
+ }
+ return serviceAccount
+}
+
+// createSubmarineStorageRBAC is a function to create RBAC for
submarine-database and submarine-minio which will be binded on service account:
submarine-storage.
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/submarine-storage-rbac.yaml
+func (r *SubmarineReconciler) createSubmarineStorageRBAC(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) error {
+ r.Log.Info("Enter createSubmarineStorageRBAC")
+
+ // Step1: Create ServiceAccount
+ serviceaccount := &corev1.ServiceAccount{}
+ err := r.Get(ctx, types.NamespacedName{Name: storageName, Namespace:
submarine.Namespace}, serviceaccount)
+
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ serviceaccount = r.newSubmarineStorageServiceAccount(ctx,
submarine)
+ err = r.Create(ctx, serviceaccount)
+ r.Log.Info("Create ServiceAccount", "name", serviceaccount.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ // Step2: Pod Security Policy if needed
+ if r.CreatePodSecurityPolicy {
+ // Step2.1: Create Role
+ role := &rbacv1.Role{}
+ err = r.Get(ctx, types.NamespacedName{Name: storageName,
Namespace: submarine.Namespace}, role)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ role = r.newSubmarineStorageRole(ctx, submarine)
+ err = r.Create(ctx, role)
+ r.Log.Info("Create Role", "name", role.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item
so we can
+ // attempt processing again later. This could have been caused
by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(role, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, role.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step2.2: Create Role Binding
+ rolebinding := &rbacv1.RoleBinding{}
+ err = r.Get(ctx, types.NamespacedName{Name: storageName,
Namespace: submarine.Namespace}, rolebinding)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ rolebinding = r.newSubmarineStorageRoleBinding(ctx,
submarine)
+ err = r.Create(ctx, rolebinding)
+ r.Log.Info("Create RoleBinding", "name",
rolebinding.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item
so we can
+ // attempt processing again later. This could have been caused
by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(rolebinding, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists,
rolebinding.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+ }
+
+ return nil
+}
diff --git a/submarine-cloud-v3/controllers/submarine_tensorboard.go
b/submarine-cloud-v3/controllers/submarine_tensorboard.go
new file mode 100644
index 00000000..898740fd
--- /dev/null
+++ b/submarine-cloud-v3/controllers/submarine_tensorboard.go
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+
+ submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
+
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *SubmarineReconciler) newSubmarineTensorboardPersistentVolumeClaim(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*corev1.PersistentVolumeClaim {
+ pvc, err := ParsePersistentVolumeClaimYaml(tensorboardYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParsePersistentVolumeClaimYaml")
+ }
+ pvc.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, pvc, r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set PersistentVolumeClaim
ControllerReference")
+ }
+ return pvc
+}
+
+func (r *SubmarineReconciler) newSubmarineTensorboardDeployment(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*appsv1.Deployment {
+ deployment, err := ParseDeploymentYaml(tensorboardYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseDeploymentYaml")
+ }
+ deployment.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, deployment,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Deployment ControllerReference")
+ }
+ return deployment
+}
+
+func (r *SubmarineReconciler) newSubmarineTensorboardService(ctx
context.Context, submarine *submarineapacheorgv1alpha1.Submarine)
*corev1.Service {
+ service, err := ParseServiceYaml(tensorboardYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseServiceYaml")
+ }
+ service.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, service,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set Service ControllerReference")
+ }
+ return service
+}
+
+// createSubmarineTensorboard is a function to create submarine-tensorboard.
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/submarine-tensorboard.yaml
+func (r *SubmarineReconciler) createSubmarineTensorboard(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) error {
+ r.Log.Info("Enter createSubmarineTensorboard")
+
+ // Step 1: Create PersistentVolumeClaim
+ pvc := &corev1.PersistentVolumeClaim{}
+ err := r.Get(ctx, types.NamespacedName{Name: tensorboardPvcName,
Namespace: submarine.Namespace}, pvc)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ pvc = r.newSubmarineTensorboardPersistentVolumeClaim(ctx,
submarine)
+ err = r.Create(ctx, pvc)
+ r.Log.Info("Create PersistentVolumeClaim", "name", pvc.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(pvc, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, pvc.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step 2: Create Deployment
+ deployment := &appsv1.Deployment{}
+ err = r.Get(ctx, types.NamespacedName{Name: tensorboardName, Namespace:
submarine.Namespace}, deployment)
+ if errors.IsNotFound(err) {
+ // If an error occurs during Get/Create, we'll requeue the item
so we can
+ // attempt processing again later. This could have been caused
by a
+ // temporary network failure, or any other transient reason.
+ deployment = r.newSubmarineTensorboardDeployment(ctx, submarine)
+ err = r.Create(ctx, deployment)
+ r.Log.Info("Create Deployment", "name", deployment.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(deployment, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, deployment.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step 3: Create Service
+ service := &corev1.Service{}
+ err = r.Get(ctx, types.NamespacedName{Name: tensorboardServiceName,
Namespace: submarine.Namespace}, service)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ service = r.newSubmarineTensorboardService(ctx, submarine)
+ err = r.Create(ctx, service)
+ r.Log.Info("Create Service", "name", service.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(service, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, service.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ return nil
+}
diff --git a/submarine-cloud-v3/controllers/submarine_virtualservice.go
b/submarine-cloud-v3/controllers/submarine_virtualservice.go
new file mode 100644
index 00000000..2158817e
--- /dev/null
+++ b/submarine-cloud-v3/controllers/submarine_virtualservice.go
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+
+ istiov1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+
+ submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
+
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *SubmarineReconciler) newSubmarineVirtualService(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) *istiov1alpha3.VirtualService {
+ virtualService, err := ParseVirtualService(virtualServiceYamlPath)
+ if err != nil {
+ r.Log.Error(err, "ParseVirtualService")
+ }
+ virtualService.Namespace = submarine.Namespace
+ err = controllerutil.SetControllerReference(submarine, virtualService,
r.Scheme)
+ if err != nil {
+ r.Log.Error(err, "Set VirtualService ControllerReference")
+ }
+ return virtualService
+}
+
+// createVirtualService is a function to create VirtualService.
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v3/artifacts/submarine-virtualservice.yaml
+func (r *SubmarineReconciler) createVirtualService(ctx context.Context,
submarine *submarineapacheorgv1alpha1.Submarine) error {
+ r.Log.Info("Enter createIngress")
+
+ virtualService := &istiov1alpha3.VirtualService{}
+ err := r.Get(ctx, types.NamespacedName{Name: virtualServiceName,
Namespace: submarine.Namespace}, virtualService)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ virtualService = r.newSubmarineVirtualService(ctx, submarine)
+ err = r.Create(ctx, virtualService)
+ r.Log.Info("Create VirtualService", "name", virtualService.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(virtualService, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, virtualService.Name)
+ r.Recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ return nil
+}
diff --git a/submarine-cloud-v3/docs/developer-guide.md
b/submarine-cloud-v3/docs/developer-guide.md
index 1bdc8154..8250f315 100644
--- a/submarine-cloud-v3/docs/developer-guide.md
+++ b/submarine-cloud-v3/docs/developer-guide.md
@@ -18,23 +18,58 @@
# Developer Guide
Golang version: `1.17`
+Kubernetes version: `1.21.0`
+
+## Prerequisites
+
+First finish the prerequisites specified in the
[QuickStart](https://submarine.apache.org/docs/next/gettingStarted/quickstart)
section on the submarine website. Prepare a minikube cluster with Istio
installed. Prepare namespaces `submarine` and `submarine-user-test` and label
them `istio-injection=enabled`.
+
+Verify with kubectl:
+
+```bash
+$ kubectl get namespace --show-labels
+NAME STATUS AGE LABELS
+istio-system Active 7d8h kubernetes.io/metadata.name=istio-system
+submarine Active 7d8h
istio-injection=enabled,kubernetes.io/metadata.name=submarine
+submarine-user-test Active 27h
istio-injection=enabled,kubernetes.io/metadata.name=submarine-user-test
+
+$ kubectl get pod -n istio-system
+NAME READY STATUS RESTARTS AGE
+istio-ingressgateway-77968dbd74-wq4vb 1/1 Running 1 7d4h
+istiod-699b647f8b-nx9rt 1/1 Running 2 7d4h
+```
+
+Next, install submarine dependencies with helm. `--set dev=true` option will
not install the operator deployment to the cluster.
+
+```bash
+helm install --set dev=true submarine ../helm-charts/submarine/ -n submarine
+```
## Run operator out-of-cluster
```bash
-# Step1: Run the operator in a terminal.
-make install run
+# Step1: Apply the submarine CRD.
+make install
+
+# Step2: Run the operator in a terminal.
+make run
-# Step2: Deploy a submarine.
-kubectl create ns submarine-user-test
+# Step3: Deploy a submarine.
kubectl apply -n submarine-user-test -f config/samples/_v1alpha1_submarine.yaml
+```
+
+If you follow the above steps, you can view the submarine workbench via the
same approach specified in the
[QuickStart](https://submarine.apache.org/docs/next/gettingStarted/quickstart)
section on the submarine website.
-# Step3: Cleanup submarine.
+
+```bash
+# Step4: Cleanup submarine.
kubectl delete -n submarine-user-test submarine example-submarine
-kubectl delete ns submarine-user-test
-# Step4: Cleanup operator.
-# Just close the running process at Step1.
+# Step5: Cleanup operator.
+# Just close the running process at Step2.
+
+# Step6: Delete the submarine CRD.
+make uninstall
```
## Run operator in-cluster
@@ -52,12 +87,12 @@ make deploy
kubectl get deployment -n submarine-cloud-v3-system
# Step4: Deploy a submarine.
-kubectl create ns submarine-user-test
kubectl apply -n submarine-user-test -f config/samples/_v1alpha1_submarine.yaml
+# You can now view the submarine workbench
+
# Step5: Cleanup submarine.
kubectl delete -n submarine-user-test submarine example-submarine
-kubectl delete ns submarine-user-test
# Step6: Cleanup operator.
make undeploy
@@ -65,7 +100,7 @@ make undeploy
### Rebuild Operator Image
-When running operator in-cluster , we need to rebuild the operator image for
changes to take effect.
+When running operator in-cluster, we need to rebuild the operator image for
changes to take effect.
```bash
eval $(minikube docker-env)
@@ -89,4 +124,12 @@ Steps to modify custom resource definition (CRD):
One can add [marker
comments](https://book.kubebuilder.io/reference/markers.html) in Go code to
control the manifests generation.
+## Add resource
+Steps to add new resource created and controlled by the operator:
+1. Add or modify yaml files under `artifacts/`.
+2. Modify `createSubmarine` in `contorllers/submarine_controller.go` to create
resource from yaml files.
+3. If needed, modify reconcile logic in `contorllers/submarine_controller.go`.
+4. If needed, import new scheme in `main.go`.
+5. If there are new resource types, add RBAC marker comments in
`contorllers/submarine_controller.go`.
+6. Run `make manifests` to update cluster role rules in
`config/rbac/role.yaml`.
diff --git a/submarine-cloud-v3/go.mod b/submarine-cloud-v3/go.mod
index 1fc55a7c..5bc44fef 100644
--- a/submarine-cloud-v3/go.mod
+++ b/submarine-cloud-v3/go.mod
@@ -3,10 +3,14 @@ module github.com/apache/submarine/submarine-cloud-v3
go 1.17
require (
+ github.com/go-logr/logr v1.2.0
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.17.0
- k8s.io/apimachinery v0.23.0
- k8s.io/client-go v0.23.0
+ github.com/pkg/errors v0.9.1
+ istio.io/client-go v1.13.4
+ k8s.io/api v0.23.1
+ k8s.io/apimachinery v0.23.1
+ k8s.io/client-go v0.23.1
sigs.k8s.io/controller-runtime v0.11.0
)
@@ -24,7 +28,6 @@ require (
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
- github.com/go-logr/logr v1.2.0 // indirect
github.com/go-logr/zapr v1.2.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da //
indirect
@@ -39,7 +42,6 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd //
indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nxadm/tail v1.4.8 // indirect
- github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.11.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.28.0 // indirect
@@ -49,7 +51,7 @@ require (
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.19.1 // indirect
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
- golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect
+ golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
@@ -62,7 +64,8 @@ require (
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
- k8s.io/api v0.23.0 // indirect
+ istio.io/api v0.0.0-20220512212136-561ffec82582 // indirect
+ istio.io/gogo-genproto v0.0.0-20211208193508-5ab4acc9eb1e // indirect
k8s.io/apiextensions-apiserver v0.23.0 // indirect
k8s.io/component-base v0.23.0 // indirect
k8s.io/klog/v2 v2.30.0 // indirect
diff --git a/submarine-cloud-v3/go.sum b/submarine-cloud-v3/go.sum
index c7f720bb..fafe8bed 100644
--- a/submarine-cloud-v3/go.sum
+++ b/submarine-cloud-v3/go.sum
@@ -97,7 +97,11 @@ github.com/client9/misspell v0.3.4/go.mod
h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod
h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod
h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod
h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod
h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod
h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod
h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod
h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod
h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod
h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
github.com/cockroachdb/errors v1.2.4/go.mod
h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod
h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
@@ -128,6 +132,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod
h1:cwu0lG7PUMfa9snN8LXBig5y
github.com/envoyproxy/go-control-plane
v0.9.9-0.20201210154907-fd9021fe5dad/go.mod
h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane
v0.9.9-0.20210217033140-668b12f5399d/go.mod
h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane
v0.9.9-0.20210512163311-63b5d3c536b0/go.mod
h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
+github.com/envoyproxy/go-control-plane
v0.9.10-0.20210907150352-cf90f659a021/go.mod
h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod
h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v0.5.2/go.mod
h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
github.com/evanphx/json-patch v4.12.0+incompatible
h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
@@ -591,8 +596,9 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod
h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod
h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod
h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod
h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210825183410-e898025ed96a
h1:bRuuGXV8wwSdGTB+CtJf+FjgO1APK1CoO39T4BN/XBw=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod
h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211209124913-491a49abca63
h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY=
+golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod
h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod
h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod
h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod
h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -867,6 +873,7 @@ google.golang.org/grpc v1.36.1/go.mod
h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG
google.golang.org/grpc v1.37.0/go.mod
h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.38.0/go.mod
h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.40.0/go.mod
h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/grpc v1.42.0/go.mod
h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod
h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod
h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod
h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -921,15 +928,24 @@ honnef.co/go/tools
v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod
h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod
h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod
h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.23.0 h1:WrL1gb73VSC8obi8cuYETJGXEoFNEh3LU0Pt+Sokgro=
+istio.io/api v0.0.0-20220512212136-561ffec82582
h1:AzLIET6ePAqxlWaXA6GOzapoRX1GRC6mZ8GY+cQIWYU=
+istio.io/api v0.0.0-20220512212136-561ffec82582/go.mod
h1:8ZZgyVgYrHhsFQarEgTfPnMGpdgTDZbxSjYhdwTUuAQ=
+istio.io/client-go v1.13.4 h1:QJBFBkOaplyL/uBL7xo75mdE5G0i1uR6BR0u9/Wuo1E=
+istio.io/client-go v1.13.4/go.mod
h1:kM3WH/HCojq7BhCD894SZuaAXUKMswT+VQRaEEhTGj0=
+istio.io/gogo-genproto v0.0.0-20211208193508-5ab4acc9eb1e
h1:z2WI3y55w0K3c6hmarcp5EcOiP4vVpTBXA8nYstP+cE=
+istio.io/gogo-genproto v0.0.0-20211208193508-5ab4acc9eb1e/go.mod
h1:vJDAniIqryf/z///fgZqVPKJ7N2lBk7Gg8DCTB7oCfU=
k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg=
+k8s.io/api v0.23.1 h1:ncu/qfBfUoClqwkTGbeRqqOqBCRoUAflMuOaOD7J0c8=
+k8s.io/api v0.23.1/go.mod h1:WfXnOnwSqNtG62Y1CdjoMxh7r7u9QXGCkA1u0na2jgo=
k8s.io/apiextensions-apiserver v0.23.0
h1:uii8BYmHYiT2ZTAJxmvc3X8UhNYMxl2A0z0Xq3Pm+WY=
k8s.io/apiextensions-apiserver v0.23.0/go.mod
h1:xIFAEEDlAZgpVBl/1VSjGDmLoXAWRG40+GsWhKhAxY4=
-k8s.io/apimachinery v0.23.0 h1:mIfWRMjBuMdolAWJ3Fd+aPTMv3X9z+waiARMpvvb0HQ=
k8s.io/apimachinery v0.23.0/go.mod
h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc=
+k8s.io/apimachinery v0.23.1 h1:sfBjlDFwj2onG0Ijx5C+SrAoeUscPrmghm7wHP+uXlo=
+k8s.io/apimachinery v0.23.1/go.mod
h1:SADt2Kl8/sttJ62RRsi9MIV4o8f5S3coArm0Iu3fBno=
k8s.io/apiserver v0.23.0/go.mod h1:Cec35u/9zAepDPPFyT+UMrgqOCjgJ5qtfVJDxjZYmt4=
-k8s.io/client-go v0.23.0 h1:vcsOqyPq7XV3QmQRCBH/t9BICJM9Q1M18qahjv+rebY=
k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA=
+k8s.io/client-go v0.23.1 h1:Ma4Fhf/p07Nmj9yAB1H7UwbFHEBrSPg8lviR24U2GiQ=
+k8s.io/client-go v0.23.1/go.mod h1:6QSI8fEuqD4zgFK0xbdwfB/PthBsIxCJMa3s17WlcO0=
k8s.io/code-generator v0.23.0/go.mod
h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE=
k8s.io/component-base v0.23.0 h1:UAnyzjvVZ2ZR1lF35YwtNY6VMN94WtOnArcXBu34es8=
k8s.io/component-base v0.23.0/go.mod
h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI=
diff --git a/submarine-cloud-v3/main.go b/submarine-cloud-v3/main.go
index 2b7de73a..7c6fde13 100644
--- a/submarine-cloud-v3/main.go
+++ b/submarine-cloud-v3/main.go
@@ -31,11 +31,33 @@ import (
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
+ istioscheme "istio.io/client-go/pkg/clientset/versioned/scheme"
+
submarineapacheorgv1alpha1
"github.com/apache/submarine/submarine-cloud-v3/api/v1alpha1"
"github.com/apache/submarine/submarine-cloud-v3/controllers"
//+kubebuilder:scaffold:imports
)
+var (
+ // Flags generated by operator-sdk
+ metricsAddr string
+ enableLeaderElection bool
+ probeAddr string
+
+ // sigs.k8s.io/controller-runtime provides the --kubeconfig flag
+ // Only required if out-of-cluster.
+ // If set, will use the kubeconfig file at that location.
+ // Otherwise will assume running in cluster and use the cluster
provided kubeconfig.
+ // kubeconfig string
+
+ // Flags of submarine
+ clusterType string
+ createPodSecurityPolicy bool
+)
+
+// Used for "source" field of events. Appears in the "FROM" column of `kubectl
describe`
+const controllerAgentName = "submarine-controller"
+
var (
scheme = runtime.NewScheme()
setupLog = ctrl.Log.WithName("setup")
@@ -43,20 +65,24 @@ var (
func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+ utilruntime.Must(istioscheme.AddToScheme(scheme))
utilruntime.Must(submarineapacheorgv1alpha1.AddToScheme(scheme))
//+kubebuilder:scaffold:scheme
}
func main() {
- var metricsAddr string
- var enableLeaderElection bool
- var probeAddr string
+ // Setup flags
+ // Flags generated by operator-sdk
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The
address the metric endpoint binds to.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The
address the probe endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active
controller manager.")
+ // Flags of submarine
+ flag.StringVar(&clusterType, "clustertype", "kubernetes", "K8s cluster
type, can be kubernetes or openshift")
+ flag.BoolVar(&createPodSecurityPolicy, "createpsp", true, "Specifies
whether a PodSecurityPolicy should be created. This configuration enables the
database/minio/server to set securityContext.runAsUser")
+
opts := zap.Options{
Development: true,
}
@@ -79,8 +105,12 @@ func main() {
}
if err = (&controllers.SubmarineReconciler{
- Client: mgr.GetClient(),
- Scheme: mgr.GetScheme(),
+ Client: mgr.GetClient(),
+ Scheme: mgr.GetScheme(),
+ Recorder:
mgr.GetEventRecorderFor(controllerAgentName),
+ Log: ctrl.Log.WithName(controllerAgentName),
+ ClusterType: clusterType,
+ CreatePodSecurityPolicy: createPodSecurityPolicy,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller",
"controller", "Submarine")
os.Exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
