This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 9bb5a761 SUBMARINE-1179. Add
PodSecurityPolicies/SecurityContextConstraints support for RunAsAnyUser in
submarine
9bb5a761 is described below
commit 9bb5a76124fac3202d36b21c42c1a46f96878e57
Author: cdmikechen <[email protected]>
AuthorDate: Sat Apr 9 15:54:33 2022 +0800
SUBMARINE-1179. Add PodSecurityPolicies/SecurityContextConstraints support
for RunAsAnyUser in submarine
### What is this PR for?
We need to add PodSecurityPolicies(k8s) or
SecurityContextConstraints(openshift) to let pod run as a user with default
user in docker container. Otherwise, pod may cause permission problems (like no
permission error).
### What type of PR is it?
Bug Fix
### Todos
* [x] - Add two params in helm values.yaml: `clusterType` and
`podSecurityPolicy.create`
* [x] - Change operator dockerfile to support shell params
`SUBMARINE_CLUSTER_TYPE` and `SUBMARINE_POD_SECURITY_POLICY_ENABLE`
* [x] - Add PodSecurityPolicy (OpenShift has a default scc anyuid so that
we need not to add)
* [x] - The processing of operator is reconstructed: create deployment run
after RBAC created
* [x] - Add RunAsAnyUser policy in database\minio\server
### What is the Jira issue?
https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-1179
### How should this be tested?
<!--
* First time? Setup Travis CI as described on
https://submarine.apache.org/contribution/contributions.html#continuous-integration
* Strongly recommended: add automated unit tests for any new or changed
behavior
* Outline any manual steps to test the PR here.
-->
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? Yes
* Does this need new documentation? No
Author: cdmikechen <[email protected]>
Signed-off-by: Kevin Su <[email protected]>
Closes #921 from cdmikechen/SUBMARINE-1179-new and squashes the following
commits:
07a005ab [cdmikechen] Fix pod startup error when minikube supports Pod
Security Policy
005f690c [cdmikechen] add submarine/finalizers in rbac
03678664 [cdmikechen] fix docker build
8fdff7d2 [cdmikechen] SUBMARINE-1179. Add
PodSecurityPolicies/SecurityContextConstraints for submarine
---
.gitignore | 4 +-
dev-support/docker-images/operator/Dockerfile | 11 +-
helm-charts/submarine/templates/_helpers.tpl | 11 ++
.../submarine/templates/{_helpers.tpl => psp.yaml} | 39 +++---
helm-charts/submarine/templates/pv.yaml | 17 +++
helm-charts/submarine/templates/rbac-kubeflow.yaml | 82 +++++++++++++
helm-charts/submarine/templates/rbac.yaml | 22 ++++
.../submarine/templates/submarine-operator.yaml | 5 +
helm-charts/submarine/values.yaml | 10 ++
submarine-cloud-v2/README.md | 2 +
.../artifacts/submarine/submarine-database.yaml | 1 +
.../artifacts/submarine/submarine-minio.yaml | 1 +
.../artifacts/submarine/submarine-mlflow.yaml | 1 +
.../submarine/submarine-storage-rbac.yaml | 40 +++---
.../artifacts/submarine/submarine-tensorboard.yaml | 1 +
submarine-cloud-v2/main.go | 16 ++-
submarine-cloud-v2/pkg/controller/controller.go | 39 +++++-
.../pkg/controller/controller_builder.go | 2 +
.../pkg/controller/controller_builder_config.go | 16 +++
.../pkg/controller/submarine_observer_rbac.go | 2 +-
.../pkg/controller/submarine_server.go | 38 +-----
.../pkg/controller/submarine_server_rbac.go | 50 +++++++-
.../pkg/controller/submarine_storage_rbac.go | 135 +++++++++++++++++++++
23 files changed, 459 insertions(+), 86 deletions(-)
diff --git a/.gitignore b/.gitignore
index 60e35b5c..312d4aa7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -86,6 +86,8 @@ submarine-cloud/bin/*
submarine-cloud-v2/vendor/*
submarine-cloud-v2/submarine-operator
submarine-cloud-v2/helm-charts/submarine-operator/charts/*
+# add operator docker build tmp
+dev-support/docker-images/operator/tmp/
# vscode file
.project
@@ -102,4 +104,4 @@ submarine-cloud-v2/helm-charts/submarine-operator/charts/*
*.log
# installation binary
-submarine-serve/installation/istioctl
\ No newline at end of file
+submarine-serve/installation/istioctl
diff --git a/dev-support/docker-images/operator/Dockerfile
b/dev-support/docker-images/operator/Dockerfile
index e2627842..3f2795a5 100644
--- a/dev-support/docker-images/operator/Dockerfile
+++ b/dev-support/docker-images/operator/Dockerfile
@@ -14,17 +14,20 @@
# limitations under the License.
FROM golang:1.16.2 AS build-image
-MAINTAINER Apache Software Foundation <[email protected]>
ADD tmp/submarine-cloud-v2 /usr/src
WORKDIR /usr/src
-RUN GOOS=linux go build -o submarine-operator
+# use CGO_ENABLED=0 to support alpine image
+RUN GOOS=linux CGO_ENABLED=0 go build -o submarine-operator
+
+# we use alpine to support shell params
+FROM alpine
+MAINTAINER Apache Software Foundation <[email protected]>
-FROM gcr.io/distroless/base-debian10
ADD tmp/submarine-cloud-v2/artifacts /usr/src/artifacts
WORKDIR /usr/src
COPY --from=build-image /usr/src/submarine-operator /usr/src/submarine-operator
-CMD ["/usr/src/submarine-operator", "-incluster=true"]
+CMD /usr/src/submarine-operator -incluster=true
-clustertype=${SUBMARINE_CLUSTER_TYPE:kubernetes}
-createpsp=${SUBMARINE_POD_SECURITY_POLICY_ENABLE:-true}
diff --git a/helm-charts/submarine/templates/_helpers.tpl
b/helm-charts/submarine/templates/_helpers.tpl
index 8f67df56..277d3fa2 100644
--- a/helm-charts/submarine/templates/_helpers.tpl
+++ b/helm-charts/submarine/templates/_helpers.tpl
@@ -31,3 +31,14 @@ parameters:
{{ end }}
{{ end }}
{{ end }}
+
+{{/*
+Return the apiVersion for PodSecurityPolicy.
+*/}}
+{{- define "podSecurityPolicy.apiVersion" -}}
+{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- print "policy/v1beta1" -}}
+{{- else -}}
+{{- print "extensions/v1beta1" -}}
+{{- end -}}
+{{- end -}}
diff --git a/helm-charts/submarine/templates/_helpers.tpl
b/helm-charts/submarine/templates/psp.yaml
similarity index 63%
copy from helm-charts/submarine/templates/_helpers.tpl
copy to helm-charts/submarine/templates/psp.yaml
index 8f67df56..dd6cf072 100644
--- a/helm-charts/submarine/templates/_helpers.tpl
+++ b/helm-charts/submarine/templates/psp.yaml
@@ -15,19 +15,26 @@
# limitations under the License.
#
-{{/*
-Set up storage class fields
-*/}}
-{{ define "storageClass.fields" }}
-{{ with .Values.storageClass }}
-reclaimPolicy: {{ .reclaimPolicy | default "Delete" }}
-volumeBindingMode: {{ .volumeBindingMode | default "Immediate" }}
-provisioner: {{ .provisioner | default "k8s.io/minikube-hostpath" }}
-{{ if .parameters }}
-parameters:
- {{ range $key, $val := .parameters }}
- {{ $key }}: {{ $val | quote }}
- {{ end }}
-{{ end }}
-{{ end }}
-{{ end }}
+{{- if .Values.podSecurityPolicy.create }}
+kind: PodSecurityPolicy
+apiVersion: {{ template "podSecurityPolicy.apiVersion" . }}
+metadata:
+ name: submarine-anyuid
+spec:
+ privileged: false
+ volumes:
+ - configMap
+ - downwardAPI
+ - emptyDir
+ - persistentVolumeClaim
+ - projected
+ - secret
+ seLinux:
+ rule: RunAsAny
+ runAsUser:
+ rule: RunAsAny
+ supplementalGroups:
+ rule: RunAsAny
+ fsGroup:
+ rule: RunAsAny
+{{- end }}
diff --git a/helm-charts/submarine/templates/pv.yaml
b/helm-charts/submarine/templates/pv.yaml
index f138ed9c..b03ae11d 100644
--- a/helm-charts/submarine/templates/pv.yaml
+++ b/helm-charts/submarine/templates/pv.yaml
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
{{- if eq .Values.storageClass.provisioner "kubernetes.io/no-provisioner"}}
apiVersion: v1
kind: PersistentVolume
diff --git a/helm-charts/submarine/templates/rbac-kubeflow.yaml
b/helm-charts/submarine/templates/rbac-kubeflow.yaml
new file mode 100644
index 00000000..00f0af33
--- /dev/null
+++ b/helm-charts/submarine/templates/rbac-kubeflow.yaml
@@ -0,0 +1,82 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+{{- if .Values.podSecurityPolicy.create }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: kubeflow-operator-anyuid
+rules:
+{{- if (eq "openshift" .Values.clusterType) }}
+ - apiGroups:
+ - security.openshift.io
+ resources:
+ - securitycontextconstraints
+ verbs:
+ - use
+ resourceNames:
+ - anyuid
+{{- else }}
+ - apiGroups:
+ - policy
+ resources:
+ - podsecuritypolicies
+ verbs:
+ - use
+ resourceNames:
+ - submarine-anyuid
+{{- end }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: tf-job-operator-anyuid
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: kubeflow-operator-anyuid
+subjects:
+ - kind: ServiceAccount
+ name: tf-job-operator
+ namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: pytorch-operator-anyuid
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: kubeflow-operator-anyuid
+subjects:
+ - kind: ServiceAccount
+ name: pytorch-operator
+ namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: notebook-controller-anyuid
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: kubeflow-operator-anyuid
+subjects:
+ - kind: ServiceAccount
+ name: notebook-controller-service-account
+ namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/helm-charts/submarine/templates/rbac.yaml
b/helm-charts/submarine/templates/rbac.yaml
index 02dec67c..2c854efb 100644
--- a/helm-charts/submarine/templates/rbac.yaml
+++ b/helm-charts/submarine/templates/rbac.yaml
@@ -25,6 +25,7 @@ rules:
resources:
- submarines
- submarines/status
+ - submarines/finalizers
verbs:
- "*"
- apiGroups:
@@ -90,6 +91,27 @@ rules:
- customresourcedefinitions/status
verbs:
- "*"
+{{- if .Values.podSecurityPolicy.create }}
+{{- if (eq "openshift" .Values.clusterType) }}
+ - apiGroups:
+ - security.openshift.io
+ resources:
+ - securitycontextconstraints
+ verbs:
+ - use
+ resourceNames:
+ - anyuid
+{{- else }}
+ - apiGroups:
+ - policy
+ resources:
+ - podsecuritypolicies
+ verbs:
+ - use
+ resourceNames:
+ - submarine-anyuid
+{{- end }}
+{{- end }}
---
apiVersion: v1
kind: ServiceAccount
diff --git a/helm-charts/submarine/templates/submarine-operator.yaml
b/helm-charts/submarine/templates/submarine-operator.yaml
index 3c7b5875..5a7b2bef 100644
--- a/helm-charts/submarine/templates/submarine-operator.yaml
+++ b/helm-charts/submarine/templates/submarine-operator.yaml
@@ -42,6 +42,11 @@ spec:
name: "{{ .Values.name }}"
resources: {}
imagePullPolicy: IfNotPresent
+ env:
+ - name: SUBMARINE_POD_SECURITY_POLICY_ENABLE
+ value: "{{ .Values.podSecurityPolicy.create }}"
+ - name: SUBMARINE_CLUSTER_TYPE
+ value: {{ .Values.clusterType }}
serviceAccountName: submarine-operator
status: {}
{{ end }}
diff --git a/helm-charts/submarine/values.yaml
b/helm-charts/submarine/values.yaml
index 534d2083..8366e5c8 100644
--- a/helm-charts/submarine/values.yaml
+++ b/helm-charts/submarine/values.yaml
@@ -30,3 +30,13 @@ storageClass:
provisioner: k8s.io/minikube-hostpath
# parameters describe volumes belonging to the storage class
parameters:
+
+# k8s cluster type. can be: kubernetes or openshift
+clusterType: kubernetes
+
+# PodSecurityPolicy configuration
+# ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/
+podSecurityPolicy:
+ # Specifies whether a PodSecurityPolicy should be created,
+ # This configuration enables the database/minio/server to set
securityContext.runAsUser
+ create: true
diff --git a/submarine-cloud-v2/README.md b/submarine-cloud-v2/README.md
index 7dbfe86f..4e4ff176 100644
--- a/submarine-cloud-v2/README.md
+++ b/submarine-cloud-v2/README.md
@@ -32,6 +32,8 @@ Golang version: `1.16.2`
go mod vendor
# Run the cluster
minikube start --vm-driver=docker --cpus 8 --memory 4096 --kubernetes-version
v1.21.2
+# Or if you want to support Pod Security Policy
(https://minikube.sigs.k8s.io/docs/tutorials/using_psp), you can use the
following command to start cluster
+minikube start
--extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy
--addons=pod-security-policy --vm-driver=docker --cpus 8 --memory 4096
--kubernetes-version v1.21.2
```
## Set up storage class fields
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml
index a6efafe7..b368e935 100644
--- a/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml
@@ -60,6 +60,7 @@ spec:
labels:
app: "submarine-database"
spec:
+ serviceAccountName: "submarine-storage"
containers:
- name: "submarine-database"
image: "apache/submarine:database-0.7.0"
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml
index a7641062..799fc096 100644
--- a/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml
@@ -57,6 +57,7 @@ spec:
labels:
app: submarine-minio
spec:
+ serviceAccountName: "submarine-storage"
containers:
- name: submarine-minio-container
image: minio/minio:RELEASE.2021-02-14T04-01-33Z
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml
index dbc513a9..dfe05ee2 100644
--- a/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml
@@ -58,6 +58,7 @@ spec:
labels:
app: submarine-mlflow
spec:
+ serviceAccountName: "submarine-storage"
initContainers:
- name: check-database-connection
image: busybox:1.28
diff --git a/helm-charts/submarine/templates/_helpers.tpl
b/submarine-cloud-v2/artifacts/submarine/submarine-storage-rbac.yaml
similarity index 62%
copy from helm-charts/submarine/templates/_helpers.tpl
copy to submarine-cloud-v2/artifacts/submarine/submarine-storage-rbac.yaml
index 8f67df56..9d4e113e 100644
--- a/helm-charts/submarine/templates/_helpers.tpl
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-storage-rbac.yaml
@@ -1,3 +1,5 @@
+---
+# Source: submarine/templates/rbac.yaml
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -15,19 +17,25 @@
# limitations under the License.
#
-{{/*
-Set up storage class fields
-*/}}
-{{ define "storageClass.fields" }}
-{{ with .Values.storageClass }}
-reclaimPolicy: {{ .reclaimPolicy | default "Delete" }}
-volumeBindingMode: {{ .volumeBindingMode | default "Immediate" }}
-provisioner: {{ .provisioner | default "k8s.io/minikube-hostpath" }}
-{{ if .parameters }}
-parameters:
- {{ range $key, $val := .parameters }}
- {{ $key }}: {{ $val | quote }}
- {{ end }}
-{{ end }}
-{{ end }}
-{{ end }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: "submarine-storage"
+rules: []
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: submarine-storage
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: "submarine-storage"
+subjects:
+ - kind: ServiceAccount
+ name: "submarine-storage"
+roleRef:
+ kind: Role
+ name: "submarine-storage"
+ apiGroup: rbac.authorization.k8s.io
diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml
b/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml
index ae60e57f..b49776b5 100644
--- a/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml
+++ b/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml
@@ -56,6 +56,7 @@ spec:
labels:
app: submarine-tensorboard
spec:
+ serviceAccountName: "submarine-storage"
containers:
- name: submarine-tensorboard-container
image: tensorflow/tensorflow:1.11.0
diff --git a/submarine-cloud-v2/main.go b/submarine-cloud-v2/main.go
index 607a12f5..486dda54 100644
--- a/submarine-cloud-v2/main.go
+++ b/submarine-cloud-v2/main.go
@@ -38,9 +38,11 @@ import (
)
var (
- masterURL string
- kubeconfig string
- incluster bool
+ masterURL string
+ kubeconfig string
+ incluster bool
+ clusterType string
+ createPodSecurityPolicy bool
)
func initKubeConfig() (*rest.Config, error) {
@@ -88,6 +90,8 @@ func main() {
// Create a Submarine operator
submarineController := NewSubmarineController(
incluster,
+ clusterType,
+ createPodSecurityPolicy,
kubeClient,
submarineClient,
traefikClient,
@@ -110,6 +114,8 @@ func main() {
func NewSubmarineController(
incluster bool,
+ clusterType string,
+ createPodSecurityPolicy bool,
kubeClient *kubernetes.Clientset,
submarineClient *clientset.Clientset,
traefikClient *traefikclientset.Clientset,
@@ -120,6 +126,8 @@ func NewSubmarineController(
bc := controller.NewControllerBuilderConfig()
bc.
InCluster(incluster).
+ WithClusterType(clusterType).
+ WithCreatePodSecurityPolicy(createPodSecurityPolicy).
WithKubeClientset(kubeClient).
WithSubmarineClientset(submarineClient).
WithTraefikClientset(traefikClient).
@@ -143,4 +151,6 @@ func init() {
flag.BoolVar(&incluster, "incluster", false, "Run submarine-operator
in-cluster")
flag.StringVar(&kubeconfig, "kubeconfig",
os.Getenv("HOME")+"/.kube/config", "Path to a kubeconfig. Only required if
out-of-cluster.")
flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes
API server. Overrides any value in kubeconfig. Only required if
out-of-cluster.")
+ flag.StringVar(&clusterType, "clustertype", "kubernetes", "K8s cluster
type, can be kubernetes or openshift")
+ flag.BoolVar(&createPodSecurityPolicy, "createpsp", true, "Specifies
whether a PodSecurityPolicy should be created. This configuration enables the
database/minio/server to set securityContext.runAsUser")
}
diff --git a/submarine-cloud-v2/pkg/controller/controller.go
b/submarine-cloud-v2/pkg/controller/controller.go
index bbb965ed..cf7bd038 100644
--- a/submarine-cloud-v2/pkg/controller/controller.go
+++ b/submarine-cloud-v2/pkg/controller/controller.go
@@ -29,6 +29,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -60,6 +61,7 @@ const (
tensorboardName = "submarine-tensorboard"
mlflowName = "submarine-mlflow"
minioName = "submarine-minio"
+ storageName = "submarine-storage"
ingressName = serverName + "-ingress"
databasePvcName = databaseName + "-pvc"
tensorboardPvcName = tensorboardName + "-pvc"
@@ -80,6 +82,7 @@ const (
tensorboardYamlPath = artifactPath +
"submarine-tensorboard.yaml"
rbacYamlPath = artifactPath + "submarine-rbac.yaml"
observerRbacYamlPath = artifactPath +
"submarine-observer-rbac.yaml"
+ storageRbacYamlPath = artifactPath +
"submarine-storage-rbac.yaml"
)
var dependents = []string{serverName, tensorboardName, mlflowName, minioName}
@@ -99,6 +102,22 @@ const (
MessageResourceSynced = "Submarine synced successfully"
)
+// Default k8s anyuid role rule
+var k8sAnyuidRoleRule = rbacv1.PolicyRule{
+ APIGroups: []string{"policy"},
+ Verbs: []string{"use"},
+ Resources: []string{"podsecuritypolicies"},
+ ResourceNames: []string{"submarine-anyuid"},
+}
+
+// Openshift anyuid role rule
+var openshiftAnyuidRoleRule = rbacv1.PolicyRule{
+ APIGroups: []string{"security.openshift.io"},
+ Verbs: []string{"use"},
+ Resources: []string{"securitycontextconstraints"},
+ ResourceNames: []string{"anyuid"},
+}
+
// Controller is the controller implementation for Submarine resources
type Controller struct {
// kubeclientset is a standard kubernetes clientset
@@ -130,7 +149,9 @@ type Controller struct {
// Kubernetes API.
recorder record.EventRecorder
- incluster bool
+ incluster bool
+ clusterType string
+ createPodSecurityPolicy bool
}
func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error {
@@ -437,27 +458,33 @@ func (c *Controller) validateSubmarine(submarine
*v1alpha1.Submarine) error {
func (c *Controller) createSubmarine(submarine *v1alpha1.Submarine) error {
var err error
- err = c.createSubmarineServer(submarine)
+ // We create rbac first, this ensures that any dependency based on it
will not go wrong
+ err = c.createSubmarineServerRBAC(submarine)
if err != nil && !errors.IsAlreadyExists(err) {
return err
}
- err = c.createSubmarineDatabase(submarine)
+ err = c.createSubmarineStorageRBAC(submarine)
if err != nil && !errors.IsAlreadyExists(err) {
return err
}
- err = c.createIngress(submarine)
+ err = c.createSubmarineObserverRBAC(submarine)
if err != nil && !errors.IsAlreadyExists(err) {
return err
}
- err = c.createSubmarineServerRBAC(submarine)
+ err = c.createSubmarineServer(submarine)
if err != nil && !errors.IsAlreadyExists(err) {
return err
}
- err = c.createSubmarineObserverRBAC(submarine)
+ err = c.createSubmarineDatabase(submarine)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ err = c.createIngress(submarine)
if err != nil && !errors.IsAlreadyExists(err) {
return err
}
diff --git a/submarine-cloud-v2/pkg/controller/controller_builder.go
b/submarine-cloud-v2/pkg/controller/controller_builder.go
index 8d3ab63f..17b8fca2 100644
--- a/submarine-cloud-v2/pkg/controller/controller_builder.go
+++ b/submarine-cloud-v2/pkg/controller/controller_builder.go
@@ -63,6 +63,8 @@ func (cb *ControllerBuilder) initialize() *ControllerBuilder {
workqueue :=
workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(),
"Submarines")
cb.controller.incluster = cb.config.incluster
+ cb.controller.clusterType = cb.config.clusterType
+ cb.controller.createPodSecurityPolicy =
cb.config.createPodSecurityPolicy
cb.controller.recorder = recorder
cb.controller.workqueue = workqueue
diff --git a/submarine-cloud-v2/pkg/controller/controller_builder_config.go
b/submarine-cloud-v2/pkg/controller/controller_builder_config.go
index 2212fb5f..337616bc 100644
--- a/submarine-cloud-v2/pkg/controller/controller_builder_config.go
+++ b/submarine-cloud-v2/pkg/controller/controller_builder_config.go
@@ -31,6 +31,8 @@ import (
type BuilderConfig struct {
incluster bool
+ clusterType string
+ createPodSecurityPolicy bool
kubeclientset kubernetes.Interface
submarineclientset clientset.Interface
traefikclientset traefik.Interface
@@ -58,6 +60,20 @@ func (bc *BuilderConfig) InCluster(
return bc
}
+func (bc *BuilderConfig) WithClusterType(
+ clusterType string,
+) *BuilderConfig {
+ bc.clusterType = clusterType
+ return bc
+}
+
+func (bc *BuilderConfig) WithCreatePodSecurityPolicy(
+ createPodSecurityPolicy bool,
+) *BuilderConfig {
+ bc.createPodSecurityPolicy = createPodSecurityPolicy
+ return bc
+}
+
func (bc *BuilderConfig) WithKubeClientset(
kubeclientset kubernetes.Interface,
) *BuilderConfig {
diff --git a/submarine-cloud-v2/pkg/controller/submarine_observer_rbac.go
b/submarine-cloud-v2/pkg/controller/submarine_observer_rbac.go
index 5f651192..b845b242 100644
--- a/submarine-cloud-v2/pkg/controller/submarine_observer_rbac.go
+++ b/submarine-cloud-v2/pkg/controller/submarine_observer_rbac.go
@@ -57,7 +57,7 @@ func newSubmarineObserverRoleBinding(submarine
*v1alpha1.Submarine) *rbacv1.Role
// createSubmarineObserverRBAC is a function to create RBAC for
submarine-observer which will be binded on service account: default.
// Reference:
https://github.com/apache/submarine/blob/master/helm-charts/submarine/templates/rbac.yaml
func (c *Controller) createSubmarineObserverRBAC(submarine
*v1alpha1.Submarine) error {
- klog.Info("[createSubmarineServerRBAC]")
+ klog.Info("[createSubmarineObserverRBAC]")
// Step1: Create Role
role, err := c.roleLister.Roles(submarine.Namespace).Get(observerName)
diff --git a/submarine-cloud-v2/pkg/controller/submarine_server.go
b/submarine-cloud-v2/pkg/controller/submarine_server.go
index e71ee0b8..a0370eaf 100644
--- a/submarine-cloud-v2/pkg/controller/submarine_server.go
+++ b/submarine-cloud-v2/pkg/controller/submarine_server.go
@@ -30,19 +30,6 @@ import (
"k8s.io/klog/v2"
)
-func newSubmarineServerServiceAccount(submarine *v1alpha1.Submarine)
*corev1.ServiceAccount {
- serviceAccount, err := ParseServiceAccountYaml(serverYamlPath)
- if err != nil {
- klog.Info("[Error] ParseServiceAccountYaml", err)
- }
-
- serviceAccount.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
- *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
- }
-
- return serviceAccount
-}
-
func newSubmarineServerService(submarine *v1alpha1.Submarine) *corev1.Service {
service, err := ParseServiceYaml(serverYamlPath)
if err != nil {
@@ -111,28 +98,7 @@ func newSubmarineServerDeployment(submarine
*v1alpha1.Submarine) *appsv1.Deploym
func (c *Controller) createSubmarineServer(submarine *v1alpha1.Submarine)
error {
klog.Info("[createSubmarineServer]")
- // Step1: Create ServiceAccount
- serviceaccount, err :=
c.serviceaccountLister.ServiceAccounts(submarine.Namespace).Get(serverName)
- // If the resource doesn't exist, we'll create it
- if errors.IsNotFound(err) {
- serviceaccount, err =
c.kubeclientset.CoreV1().ServiceAccounts(submarine.Namespace).Create(context.TODO(),
newSubmarineServerServiceAccount(submarine), metav1.CreateOptions{})
- klog.Info(" Create ServiceAccount: ", serviceaccount.Name)
- }
-
- // If an error occurs during Get/Create, we'll requeue the item so we
can
- // attempt processing again later. This could have been caused by a
- // temporary network failure, or any other transient reason.
- if err != nil {
- return err
- }
-
- if !metav1.IsControlledBy(serviceaccount, submarine) {
- msg := fmt.Sprintf(MessageResourceExists, serviceaccount.Name)
- c.recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
- return fmt.Errorf(msg)
- }
-
- // Step2: Create Service
+ // Step1: Create Service
service, err :=
c.serviceLister.Services(submarine.Namespace).Get(serverName)
// If the resource doesn't exist, we'll create it
if errors.IsNotFound(err) {
@@ -153,7 +119,7 @@ func (c *Controller) createSubmarineServer(submarine
*v1alpha1.Submarine) error
return fmt.Errorf(msg)
}
- // Step3: Create Deployment
+ // Step2: Create Deployment
deployment, err :=
c.deploymentLister.Deployments(submarine.Namespace).Get(serverName)
// If the resource doesn't exist, we'll create it
if errors.IsNotFound(err) {
diff --git a/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go
b/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go
index 61e0bb81..12268a26 100644
--- a/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go
+++ b/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go
@@ -30,7 +30,20 @@ import (
"k8s.io/klog/v2"
)
-func newSubmarineServerRole(submarine *v1alpha1.Submarine) *rbacv1.Role {
+func newSubmarineServerServiceAccount(submarine *v1alpha1.Submarine)
*corev1.ServiceAccount {
+ serviceAccount, err := ParseServiceAccountYaml(serverYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseServiceAccountYaml", err)
+ }
+
+ serviceAccount.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
+ }
+
+ return serviceAccount
+}
+
+func newSubmarineServerRole(c *Controller, submarine *v1alpha1.Submarine)
*rbacv1.Role {
role, err := ParseRoleYaml(rbacYamlPath)
if err != nil {
klog.Info("[Error] ParseRole", err)
@@ -39,6 +52,15 @@ func newSubmarineServerRole(submarine *v1alpha1.Submarine)
*rbacv1.Role {
*metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
}
+ if c.createPodSecurityPolicy {
+ // If cluster type is openshift and need create pod security
policy, we need add anyuid scc, or we add k8s psp
+ if c.clusterType == "openshift" {
+ role.Rules = append(role.Rules, openshiftAnyuidRoleRule)
+ } else {
+ role.Rules = append(role.Rules, k8sAnyuidRoleRule)
+ }
+ }
+
return role
}
@@ -59,11 +81,32 @@ func newSubmarineServerRoleBinding(submarine
*v1alpha1.Submarine) *rbacv1.RoleBi
func (c *Controller) createSubmarineServerRBAC(submarine *v1alpha1.Submarine)
error {
klog.Info("[createSubmarineServerRBAC]")
- // Step1: Create Role
+ // Step1: Create ServiceAccount
+ serviceaccount, err :=
c.serviceaccountLister.ServiceAccounts(submarine.Namespace).Get(serverName)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ serviceaccount, err =
c.kubeclientset.CoreV1().ServiceAccounts(submarine.Namespace).Create(context.TODO(),
newSubmarineServerServiceAccount(submarine), metav1.CreateOptions{})
+ klog.Info(" Create ServiceAccount: ", serviceaccount.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item so we
can
+ // attempt processing again later. This could have been caused by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(serviceaccount, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, serviceaccount.Name)
+ c.recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step2: Create Role
role, err := c.roleLister.Roles(submarine.Namespace).Get(serverName)
// If the resource doesn't exist, we'll create it
if errors.IsNotFound(err) {
- role, err =
c.kubeclientset.RbacV1().Roles(submarine.Namespace).Create(context.TODO(),
newSubmarineServerRole(submarine), metav1.CreateOptions{})
+ role, err =
c.kubeclientset.RbacV1().Roles(submarine.Namespace).Create(context.TODO(),
newSubmarineServerRole(c, submarine), metav1.CreateOptions{})
klog.Info(" Create Role: ", role.Name)
}
@@ -80,6 +123,7 @@ func (c *Controller) createSubmarineServerRBAC(submarine
*v1alpha1.Submarine) er
return fmt.Errorf(msg)
}
+ // Step3: Create Role Binding
rolebinding, rolebinding_err :=
c.rolebindingLister.RoleBindings(submarine.Namespace).Get(serverName)
// If the resource doesn't exist, we'll create it
if errors.IsNotFound(rolebinding_err) {
diff --git a/submarine-cloud-v2/pkg/controller/submarine_storage_rbac.go
b/submarine-cloud-v2/pkg/controller/submarine_storage_rbac.go
new file mode 100644
index 00000000..05a38adf
--- /dev/null
+++ b/submarine-cloud-v2/pkg/controller/submarine_storage_rbac.go
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controller
+
+import (
+ "context"
+ "fmt"
+ v1alpha1
"github.com/apache/submarine/submarine-cloud-v2/pkg/apis/submarine/v1alpha1"
+
+ corev1 "k8s.io/api/core/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/klog/v2"
+)
+
+func newSubmarineStorageRole(c *Controller, submarine *v1alpha1.Submarine)
*rbacv1.Role {
+ role, err := ParseRoleYaml(storageRbacYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseRole", err)
+ }
+ role.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
+ }
+
+ // If cluster type is openshift and need create pod security policy, we
need add anyuid scc, or we add k8s psp
+ if c.clusterType == "openshift" {
+ role.Rules = append(role.Rules, openshiftAnyuidRoleRule)
+ } else {
+ role.Rules = append(role.Rules, k8sAnyuidRoleRule)
+ }
+
+ return role
+}
+
+func newSubmarineStorageRoleBinding(submarine *v1alpha1.Submarine)
*rbacv1.RoleBinding {
+ roleBinding, err := ParseRoleBindingYaml(storageRbacYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseRoleBinding", err)
+ }
+ roleBinding.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
+ }
+
+ return roleBinding
+}
+
+func newSubmarineStorageServiceAccount(submarine *v1alpha1.Submarine)
*corev1.ServiceAccount {
+ serviceAccount, err := ParseServiceAccountYaml(storageRbacYamlPath)
+ if err != nil {
+ klog.Info("[Error] ParseServiceAccountYaml", err)
+ }
+
+ serviceAccount.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+ *metav1.NewControllerRef(submarine,
v1alpha1.SchemeGroupVersion.WithKind("Submarine")),
+ }
+
+ return serviceAccount
+}
+
+// createSubmarineStorageRBAC is a function to create RBAC for
submarine-database and submarine-minio which will be binded on service account:
submarine-storage.
+// Reference:
https://github.com/apache/submarine/blob/master/submarine-cloud-v2/artifacts/submarine/submarine-storage-rbac.yaml
+func (c *Controller) createSubmarineStorageRBAC(submarine *v1alpha1.Submarine)
error {
+ klog.Info("[createSubmarineStorageRBAC]")
+
+ // Step1: Create ServiceAccount
+ serviceaccount, err :=
c.serviceaccountLister.ServiceAccounts(submarine.Namespace).Get(storageName)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ serviceaccount, err =
c.kubeclientset.CoreV1().ServiceAccounts(submarine.Namespace).Create(context.TODO(),
newSubmarineStorageServiceAccount(submarine), metav1.CreateOptions{})
+ klog.Info(" Create ServiceAccount: ", serviceaccount.Name)
+ }
+
+ // Step2: Pod Security Policy if needed
+ if c.createPodSecurityPolicy {
+ // Step2.1: Create Role
+ role, err :=
c.roleLister.Roles(submarine.Namespace).Get(storageName)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(err) {
+ role, err =
c.kubeclientset.RbacV1().Roles(submarine.Namespace).Create(context.TODO(),
newSubmarineStorageRole(c, submarine), metav1.CreateOptions{})
+ klog.Info(" Create Role: ", role.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item
so we can
+ // attempt processing again later. This could have been caused
by a
+ // temporary network failure, or any other transient reason.
+ if err != nil {
+ return err
+ }
+
+ if !metav1.IsControlledBy(role, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists, role.Name)
+ c.recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+
+ // Step2.2: Create Role Binding
+ rolebinding, rolebinding_err :=
c.rolebindingLister.RoleBindings(submarine.Namespace).Get(storageName)
+ // If the resource doesn't exist, we'll create it
+ if errors.IsNotFound(rolebinding_err) {
+ rolebinding, rolebinding_err =
c.kubeclientset.RbacV1().RoleBindings(submarine.Namespace).Create(context.TODO(),
newSubmarineStorageRoleBinding(submarine), metav1.CreateOptions{})
+ klog.Info(" Create RoleBinding: ", rolebinding.Name)
+ }
+
+ // If an error occurs during Get/Create, we'll requeue the item
so we can
+ // attempt processing again later. This could have been caused
by a
+ // temporary network failure, or any other transient reason.
+ if rolebinding_err != nil {
+ return rolebinding_err
+ }
+
+ if !metav1.IsControlledBy(rolebinding, submarine) {
+ msg := fmt.Sprintf(MessageResourceExists,
rolebinding.Name)
+ c.recorder.Event(submarine, corev1.EventTypeWarning,
ErrResourceExists, msg)
+ return fmt.Errorf(msg)
+ }
+ }
+
+ return nil
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
