This is an automated email from the ASF dual-hosted git repository. pingsutw pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push: new 9bb5a761 SUBMARINE-1179. Add PodSecurityPolicies/SecurityContextConstraints support for RunAsAnyUser in submarine 9bb5a761 is described below commit 9bb5a76124fac3202d36b21c42c1a46f96878e57 Author: cdmikechen <cdmikec...@hotmail.com> AuthorDate: Sat Apr 9 15:54:33 2022 +0800 SUBMARINE-1179. Add PodSecurityPolicies/SecurityContextConstraints support for RunAsAnyUser in submarine ### What is this PR for? We need to add PodSecurityPolicies(k8s) or SecurityContextConstraints(openshift) to let pod run as a user with default user in docker container. Otherwise, pod may cause permission problems (like no permission error). ### What type of PR is it? Bug Fix ### Todos * [x] - Add two params in helm values.yaml: `clusterType` and `podSecurityPolicy.create` * [x] - Change operator dockerfile to support shell params `SUBMARINE_CLUSTER_TYPE` and `SUBMARINE_POD_SECURITY_POLICY_ENABLE` * [x] - Add PodSecurityPolicy (OpenShift has a default scc anyuid so that we need not to add) * [x] - The processing of operator is reconstructed: create deployment run after RBAC created * [x] - Add RunAsAnyUser policy in database\minio\server ### What is the Jira issue? https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-1179 ### How should this be tested? <!-- * First time? Setup Travis CI as described on https://submarine.apache.org/contribution/contributions.html#continuous-integration * Strongly recommended: add automated unit tests for any new or changed behavior * Outline any manual steps to test the PR here. --> ### Screenshots (if appropriate) ### Questions: * Do the license files need updating? No * Are there breaking changes for older versions? Yes * Does this need new documentation? No Author: cdmikechen <cdmikec...@hotmail.com> Signed-off-by: Kevin Su <pings...@apache.org> Closes #921 from cdmikechen/SUBMARINE-1179-new and squashes the following commits: 07a005ab [cdmikechen] Fix pod startup error when minikube supports Pod Security Policy 005f690c [cdmikechen] add submarine/finalizers in rbac 03678664 [cdmikechen] fix docker build 8fdff7d2 [cdmikechen] SUBMARINE-1179. Add PodSecurityPolicies/SecurityContextConstraints for submarine --- .gitignore | 4 +- dev-support/docker-images/operator/Dockerfile | 11 +- helm-charts/submarine/templates/_helpers.tpl | 11 ++ .../submarine/templates/{_helpers.tpl => psp.yaml} | 39 +++--- helm-charts/submarine/templates/pv.yaml | 17 +++ helm-charts/submarine/templates/rbac-kubeflow.yaml | 82 +++++++++++++ helm-charts/submarine/templates/rbac.yaml | 22 ++++ .../submarine/templates/submarine-operator.yaml | 5 + helm-charts/submarine/values.yaml | 10 ++ submarine-cloud-v2/README.md | 2 + .../artifacts/submarine/submarine-database.yaml | 1 + .../artifacts/submarine/submarine-minio.yaml | 1 + .../artifacts/submarine/submarine-mlflow.yaml | 1 + .../submarine/submarine-storage-rbac.yaml | 40 +++--- .../artifacts/submarine/submarine-tensorboard.yaml | 1 + submarine-cloud-v2/main.go | 16 ++- submarine-cloud-v2/pkg/controller/controller.go | 39 +++++- .../pkg/controller/controller_builder.go | 2 + .../pkg/controller/controller_builder_config.go | 16 +++ .../pkg/controller/submarine_observer_rbac.go | 2 +- .../pkg/controller/submarine_server.go | 38 +----- .../pkg/controller/submarine_server_rbac.go | 50 +++++++- .../pkg/controller/submarine_storage_rbac.go | 135 +++++++++++++++++++++ 23 files changed, 459 insertions(+), 86 deletions(-) diff --git a/.gitignore b/.gitignore index 60e35b5c..312d4aa7 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,8 @@ submarine-cloud/bin/* submarine-cloud-v2/vendor/* submarine-cloud-v2/submarine-operator submarine-cloud-v2/helm-charts/submarine-operator/charts/* +# add operator docker build tmp +dev-support/docker-images/operator/tmp/ # vscode file .project @@ -102,4 +104,4 @@ submarine-cloud-v2/helm-charts/submarine-operator/charts/* *.log # installation binary -submarine-serve/installation/istioctl \ No newline at end of file +submarine-serve/installation/istioctl diff --git a/dev-support/docker-images/operator/Dockerfile b/dev-support/docker-images/operator/Dockerfile index e2627842..3f2795a5 100644 --- a/dev-support/docker-images/operator/Dockerfile +++ b/dev-support/docker-images/operator/Dockerfile @@ -14,17 +14,20 @@ # limitations under the License. FROM golang:1.16.2 AS build-image -MAINTAINER Apache Software Foundation <dev@submarine.apache.org> ADD tmp/submarine-cloud-v2 /usr/src WORKDIR /usr/src -RUN GOOS=linux go build -o submarine-operator +# use CGO_ENABLED=0 to support alpine image +RUN GOOS=linux CGO_ENABLED=0 go build -o submarine-operator + +# we use alpine to support shell params +FROM alpine +MAINTAINER Apache Software Foundation <dev@submarine.apache.org> -FROM gcr.io/distroless/base-debian10 ADD tmp/submarine-cloud-v2/artifacts /usr/src/artifacts WORKDIR /usr/src COPY --from=build-image /usr/src/submarine-operator /usr/src/submarine-operator -CMD ["/usr/src/submarine-operator", "-incluster=true"] +CMD /usr/src/submarine-operator -incluster=true -clustertype=${SUBMARINE_CLUSTER_TYPE:kubernetes} -createpsp=${SUBMARINE_POD_SECURITY_POLICY_ENABLE:-true} diff --git a/helm-charts/submarine/templates/_helpers.tpl b/helm-charts/submarine/templates/_helpers.tpl index 8f67df56..277d3fa2 100644 --- a/helm-charts/submarine/templates/_helpers.tpl +++ b/helm-charts/submarine/templates/_helpers.tpl @@ -31,3 +31,14 @@ parameters: {{ end }} {{ end }} {{ end }} + +{{/* +Return the apiVersion for PodSecurityPolicy. +*/}} +{{- define "podSecurityPolicy.apiVersion" -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "extensions/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/helm-charts/submarine/templates/_helpers.tpl b/helm-charts/submarine/templates/psp.yaml similarity index 63% copy from helm-charts/submarine/templates/_helpers.tpl copy to helm-charts/submarine/templates/psp.yaml index 8f67df56..dd6cf072 100644 --- a/helm-charts/submarine/templates/_helpers.tpl +++ b/helm-charts/submarine/templates/psp.yaml @@ -15,19 +15,26 @@ # limitations under the License. # -{{/* -Set up storage class fields -*/}} -{{ define "storageClass.fields" }} -{{ with .Values.storageClass }} -reclaimPolicy: {{ .reclaimPolicy | default "Delete" }} -volumeBindingMode: {{ .volumeBindingMode | default "Immediate" }} -provisioner: {{ .provisioner | default "k8s.io/minikube-hostpath" }} -{{ if .parameters }} -parameters: - {{ range $key, $val := .parameters }} - {{ $key }}: {{ $val | quote }} - {{ end }} -{{ end }} -{{ end }} -{{ end }} +{{- if .Values.podSecurityPolicy.create }} +kind: PodSecurityPolicy +apiVersion: {{ template "podSecurityPolicy.apiVersion" . }} +metadata: + name: submarine-anyuid +spec: + privileged: false + volumes: + - configMap + - downwardAPI + - emptyDir + - persistentVolumeClaim + - projected + - secret + seLinux: + rule: RunAsAny + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + fsGroup: + rule: RunAsAny +{{- end }} diff --git a/helm-charts/submarine/templates/pv.yaml b/helm-charts/submarine/templates/pv.yaml index f138ed9c..b03ae11d 100644 --- a/helm-charts/submarine/templates/pv.yaml +++ b/helm-charts/submarine/templates/pv.yaml @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + {{- if eq .Values.storageClass.provisioner "kubernetes.io/no-provisioner"}} apiVersion: v1 kind: PersistentVolume diff --git a/helm-charts/submarine/templates/rbac-kubeflow.yaml b/helm-charts/submarine/templates/rbac-kubeflow.yaml new file mode 100644 index 00000000..00f0af33 --- /dev/null +++ b/helm-charts/submarine/templates/rbac-kubeflow.yaml @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.podSecurityPolicy.create }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubeflow-operator-anyuid +rules: +{{- if (eq "openshift" .Values.clusterType) }} + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - use + resourceNames: + - anyuid +{{- else }} + - apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - submarine-anyuid +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: tf-job-operator-anyuid +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflow-operator-anyuid +subjects: + - kind: ServiceAccount + name: tf-job-operator + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pytorch-operator-anyuid +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflow-operator-anyuid +subjects: + - kind: ServiceAccount + name: pytorch-operator + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: notebook-controller-anyuid +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflow-operator-anyuid +subjects: + - kind: ServiceAccount + name: notebook-controller-service-account + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/helm-charts/submarine/templates/rbac.yaml b/helm-charts/submarine/templates/rbac.yaml index 02dec67c..2c854efb 100644 --- a/helm-charts/submarine/templates/rbac.yaml +++ b/helm-charts/submarine/templates/rbac.yaml @@ -25,6 +25,7 @@ rules: resources: - submarines - submarines/status + - submarines/finalizers verbs: - "*" - apiGroups: @@ -90,6 +91,27 @@ rules: - customresourcedefinitions/status verbs: - "*" +{{- if .Values.podSecurityPolicy.create }} +{{- if (eq "openshift" .Values.clusterType) }} + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - use + resourceNames: + - anyuid +{{- else }} + - apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - submarine-anyuid +{{- end }} +{{- end }} --- apiVersion: v1 kind: ServiceAccount diff --git a/helm-charts/submarine/templates/submarine-operator.yaml b/helm-charts/submarine/templates/submarine-operator.yaml index 3c7b5875..5a7b2bef 100644 --- a/helm-charts/submarine/templates/submarine-operator.yaml +++ b/helm-charts/submarine/templates/submarine-operator.yaml @@ -42,6 +42,11 @@ spec: name: "{{ .Values.name }}" resources: {} imagePullPolicy: IfNotPresent + env: + - name: SUBMARINE_POD_SECURITY_POLICY_ENABLE + value: "{{ .Values.podSecurityPolicy.create }}" + - name: SUBMARINE_CLUSTER_TYPE + value: {{ .Values.clusterType }} serviceAccountName: submarine-operator status: {} {{ end }} diff --git a/helm-charts/submarine/values.yaml b/helm-charts/submarine/values.yaml index 534d2083..8366e5c8 100644 --- a/helm-charts/submarine/values.yaml +++ b/helm-charts/submarine/values.yaml @@ -30,3 +30,13 @@ storageClass: provisioner: k8s.io/minikube-hostpath # parameters describe volumes belonging to the storage class parameters: + +# k8s cluster type. can be: kubernetes or openshift +clusterType: kubernetes + +# PodSecurityPolicy configuration +# ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +podSecurityPolicy: + # Specifies whether a PodSecurityPolicy should be created, + # This configuration enables the database/minio/server to set securityContext.runAsUser + create: true diff --git a/submarine-cloud-v2/README.md b/submarine-cloud-v2/README.md index 7dbfe86f..4e4ff176 100644 --- a/submarine-cloud-v2/README.md +++ b/submarine-cloud-v2/README.md @@ -32,6 +32,8 @@ Golang version: `1.16.2` go mod vendor # Run the cluster minikube start --vm-driver=docker --cpus 8 --memory 4096 --kubernetes-version v1.21.2 +# Or if you want to support Pod Security Policy (https://minikube.sigs.k8s.io/docs/tutorials/using_psp), you can use the following command to start cluster +minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy --addons=pod-security-policy --vm-driver=docker --cpus 8 --memory 4096 --kubernetes-version v1.21.2 ``` ## Set up storage class fields diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml b/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml index a6efafe7..b368e935 100644 --- a/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml +++ b/submarine-cloud-v2/artifacts/submarine/submarine-database.yaml @@ -60,6 +60,7 @@ spec: labels: app: "submarine-database" spec: + serviceAccountName: "submarine-storage" containers: - name: "submarine-database" image: "apache/submarine:database-0.7.0" diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml b/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml index a7641062..799fc096 100644 --- a/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml +++ b/submarine-cloud-v2/artifacts/submarine/submarine-minio.yaml @@ -57,6 +57,7 @@ spec: labels: app: submarine-minio spec: + serviceAccountName: "submarine-storage" containers: - name: submarine-minio-container image: minio/minio:RELEASE.2021-02-14T04-01-33Z diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml b/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml index dbc513a9..dfe05ee2 100644 --- a/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml +++ b/submarine-cloud-v2/artifacts/submarine/submarine-mlflow.yaml @@ -58,6 +58,7 @@ spec: labels: app: submarine-mlflow spec: + serviceAccountName: "submarine-storage" initContainers: - name: check-database-connection image: busybox:1.28 diff --git a/helm-charts/submarine/templates/_helpers.tpl b/submarine-cloud-v2/artifacts/submarine/submarine-storage-rbac.yaml similarity index 62% copy from helm-charts/submarine/templates/_helpers.tpl copy to submarine-cloud-v2/artifacts/submarine/submarine-storage-rbac.yaml index 8f67df56..9d4e113e 100644 --- a/helm-charts/submarine/templates/_helpers.tpl +++ b/submarine-cloud-v2/artifacts/submarine/submarine-storage-rbac.yaml @@ -1,3 +1,5 @@ +--- +# Source: submarine/templates/rbac.yaml # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -15,19 +17,25 @@ # limitations under the License. # -{{/* -Set up storage class fields -*/}} -{{ define "storageClass.fields" }} -{{ with .Values.storageClass }} -reclaimPolicy: {{ .reclaimPolicy | default "Delete" }} -volumeBindingMode: {{ .volumeBindingMode | default "Immediate" }} -provisioner: {{ .provisioner | default "k8s.io/minikube-hostpath" }} -{{ if .parameters }} -parameters: - {{ range $key, $val := .parameters }} - {{ $key }}: {{ $val | quote }} - {{ end }} -{{ end }} -{{ end }} -{{ end }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: "submarine-storage" +rules: [] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: submarine-storage +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: "submarine-storage" +subjects: + - kind: ServiceAccount + name: "submarine-storage" +roleRef: + kind: Role + name: "submarine-storage" + apiGroup: rbac.authorization.k8s.io diff --git a/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml b/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml index ae60e57f..b49776b5 100644 --- a/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml +++ b/submarine-cloud-v2/artifacts/submarine/submarine-tensorboard.yaml @@ -56,6 +56,7 @@ spec: labels: app: submarine-tensorboard spec: + serviceAccountName: "submarine-storage" containers: - name: submarine-tensorboard-container image: tensorflow/tensorflow:1.11.0 diff --git a/submarine-cloud-v2/main.go b/submarine-cloud-v2/main.go index 607a12f5..486dda54 100644 --- a/submarine-cloud-v2/main.go +++ b/submarine-cloud-v2/main.go @@ -38,9 +38,11 @@ import ( ) var ( - masterURL string - kubeconfig string - incluster bool + masterURL string + kubeconfig string + incluster bool + clusterType string + createPodSecurityPolicy bool ) func initKubeConfig() (*rest.Config, error) { @@ -88,6 +90,8 @@ func main() { // Create a Submarine operator submarineController := NewSubmarineController( incluster, + clusterType, + createPodSecurityPolicy, kubeClient, submarineClient, traefikClient, @@ -110,6 +114,8 @@ func main() { func NewSubmarineController( incluster bool, + clusterType string, + createPodSecurityPolicy bool, kubeClient *kubernetes.Clientset, submarineClient *clientset.Clientset, traefikClient *traefikclientset.Clientset, @@ -120,6 +126,8 @@ func NewSubmarineController( bc := controller.NewControllerBuilderConfig() bc. InCluster(incluster). + WithClusterType(clusterType). + WithCreatePodSecurityPolicy(createPodSecurityPolicy). WithKubeClientset(kubeClient). WithSubmarineClientset(submarineClient). WithTraefikClientset(traefikClient). @@ -143,4 +151,6 @@ func init() { flag.BoolVar(&incluster, "incluster", false, "Run submarine-operator in-cluster") flag.StringVar(&kubeconfig, "kubeconfig", os.Getenv("HOME")+"/.kube/config", "Path to a kubeconfig. Only required if out-of-cluster.") flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") + flag.StringVar(&clusterType, "clustertype", "kubernetes", "K8s cluster type, can be kubernetes or openshift") + flag.BoolVar(&createPodSecurityPolicy, "createpsp", true, "Specifies whether a PodSecurityPolicy should be created. This configuration enables the database/minio/server to set securityContext.runAsUser") } diff --git a/submarine-cloud-v2/pkg/controller/controller.go b/submarine-cloud-v2/pkg/controller/controller.go index bbb965ed..cf7bd038 100644 --- a/submarine-cloud-v2/pkg/controller/controller.go +++ b/submarine-cloud-v2/pkg/controller/controller.go @@ -29,6 +29,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -60,6 +61,7 @@ const ( tensorboardName = "submarine-tensorboard" mlflowName = "submarine-mlflow" minioName = "submarine-minio" + storageName = "submarine-storage" ingressName = serverName + "-ingress" databasePvcName = databaseName + "-pvc" tensorboardPvcName = tensorboardName + "-pvc" @@ -80,6 +82,7 @@ const ( tensorboardYamlPath = artifactPath + "submarine-tensorboard.yaml" rbacYamlPath = artifactPath + "submarine-rbac.yaml" observerRbacYamlPath = artifactPath + "submarine-observer-rbac.yaml" + storageRbacYamlPath = artifactPath + "submarine-storage-rbac.yaml" ) var dependents = []string{serverName, tensorboardName, mlflowName, minioName} @@ -99,6 +102,22 @@ const ( MessageResourceSynced = "Submarine synced successfully" ) +// Default k8s anyuid role rule +var k8sAnyuidRoleRule = rbacv1.PolicyRule{ + APIGroups: []string{"policy"}, + Verbs: []string{"use"}, + Resources: []string{"podsecuritypolicies"}, + ResourceNames: []string{"submarine-anyuid"}, +} + +// Openshift anyuid role rule +var openshiftAnyuidRoleRule = rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Verbs: []string{"use"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{"anyuid"}, +} + // Controller is the controller implementation for Submarine resources type Controller struct { // kubeclientset is a standard kubernetes clientset @@ -130,7 +149,9 @@ type Controller struct { // Kubernetes API. recorder record.EventRecorder - incluster bool + incluster bool + clusterType string + createPodSecurityPolicy bool } func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { @@ -437,27 +458,33 @@ func (c *Controller) validateSubmarine(submarine *v1alpha1.Submarine) error { func (c *Controller) createSubmarine(submarine *v1alpha1.Submarine) error { var err error - err = c.createSubmarineServer(submarine) + // We create rbac first, this ensures that any dependency based on it will not go wrong + err = c.createSubmarineServerRBAC(submarine) if err != nil && !errors.IsAlreadyExists(err) { return err } - err = c.createSubmarineDatabase(submarine) + err = c.createSubmarineStorageRBAC(submarine) if err != nil && !errors.IsAlreadyExists(err) { return err } - err = c.createIngress(submarine) + err = c.createSubmarineObserverRBAC(submarine) if err != nil && !errors.IsAlreadyExists(err) { return err } - err = c.createSubmarineServerRBAC(submarine) + err = c.createSubmarineServer(submarine) if err != nil && !errors.IsAlreadyExists(err) { return err } - err = c.createSubmarineObserverRBAC(submarine) + err = c.createSubmarineDatabase(submarine) + if err != nil && !errors.IsAlreadyExists(err) { + return err + } + + err = c.createIngress(submarine) if err != nil && !errors.IsAlreadyExists(err) { return err } diff --git a/submarine-cloud-v2/pkg/controller/controller_builder.go b/submarine-cloud-v2/pkg/controller/controller_builder.go index 8d3ab63f..17b8fca2 100644 --- a/submarine-cloud-v2/pkg/controller/controller_builder.go +++ b/submarine-cloud-v2/pkg/controller/controller_builder.go @@ -63,6 +63,8 @@ func (cb *ControllerBuilder) initialize() *ControllerBuilder { workqueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Submarines") cb.controller.incluster = cb.config.incluster + cb.controller.clusterType = cb.config.clusterType + cb.controller.createPodSecurityPolicy = cb.config.createPodSecurityPolicy cb.controller.recorder = recorder cb.controller.workqueue = workqueue diff --git a/submarine-cloud-v2/pkg/controller/controller_builder_config.go b/submarine-cloud-v2/pkg/controller/controller_builder_config.go index 2212fb5f..337616bc 100644 --- a/submarine-cloud-v2/pkg/controller/controller_builder_config.go +++ b/submarine-cloud-v2/pkg/controller/controller_builder_config.go @@ -31,6 +31,8 @@ import ( type BuilderConfig struct { incluster bool + clusterType string + createPodSecurityPolicy bool kubeclientset kubernetes.Interface submarineclientset clientset.Interface traefikclientset traefik.Interface @@ -58,6 +60,20 @@ func (bc *BuilderConfig) InCluster( return bc } +func (bc *BuilderConfig) WithClusterType( + clusterType string, +) *BuilderConfig { + bc.clusterType = clusterType + return bc +} + +func (bc *BuilderConfig) WithCreatePodSecurityPolicy( + createPodSecurityPolicy bool, +) *BuilderConfig { + bc.createPodSecurityPolicy = createPodSecurityPolicy + return bc +} + func (bc *BuilderConfig) WithKubeClientset( kubeclientset kubernetes.Interface, ) *BuilderConfig { diff --git a/submarine-cloud-v2/pkg/controller/submarine_observer_rbac.go b/submarine-cloud-v2/pkg/controller/submarine_observer_rbac.go index 5f651192..b845b242 100644 --- a/submarine-cloud-v2/pkg/controller/submarine_observer_rbac.go +++ b/submarine-cloud-v2/pkg/controller/submarine_observer_rbac.go @@ -57,7 +57,7 @@ func newSubmarineObserverRoleBinding(submarine *v1alpha1.Submarine) *rbacv1.Role // createSubmarineObserverRBAC is a function to create RBAC for submarine-observer which will be binded on service account: default. // Reference: https://github.com/apache/submarine/blob/master/helm-charts/submarine/templates/rbac.yaml func (c *Controller) createSubmarineObserverRBAC(submarine *v1alpha1.Submarine) error { - klog.Info("[createSubmarineServerRBAC]") + klog.Info("[createSubmarineObserverRBAC]") // Step1: Create Role role, err := c.roleLister.Roles(submarine.Namespace).Get(observerName) diff --git a/submarine-cloud-v2/pkg/controller/submarine_server.go b/submarine-cloud-v2/pkg/controller/submarine_server.go index e71ee0b8..a0370eaf 100644 --- a/submarine-cloud-v2/pkg/controller/submarine_server.go +++ b/submarine-cloud-v2/pkg/controller/submarine_server.go @@ -30,19 +30,6 @@ import ( "k8s.io/klog/v2" ) -func newSubmarineServerServiceAccount(submarine *v1alpha1.Submarine) *corev1.ServiceAccount { - serviceAccount, err := ParseServiceAccountYaml(serverYamlPath) - if err != nil { - klog.Info("[Error] ParseServiceAccountYaml", err) - } - - serviceAccount.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ - *metav1.NewControllerRef(submarine, v1alpha1.SchemeGroupVersion.WithKind("Submarine")), - } - - return serviceAccount -} - func newSubmarineServerService(submarine *v1alpha1.Submarine) *corev1.Service { service, err := ParseServiceYaml(serverYamlPath) if err != nil { @@ -111,28 +98,7 @@ func newSubmarineServerDeployment(submarine *v1alpha1.Submarine) *appsv1.Deploym func (c *Controller) createSubmarineServer(submarine *v1alpha1.Submarine) error { klog.Info("[createSubmarineServer]") - // Step1: Create ServiceAccount - serviceaccount, err := c.serviceaccountLister.ServiceAccounts(submarine.Namespace).Get(serverName) - // If the resource doesn't exist, we'll create it - if errors.IsNotFound(err) { - serviceaccount, err = c.kubeclientset.CoreV1().ServiceAccounts(submarine.Namespace).Create(context.TODO(), newSubmarineServerServiceAccount(submarine), metav1.CreateOptions{}) - klog.Info(" Create ServiceAccount: ", serviceaccount.Name) - } - - // If an error occurs during Get/Create, we'll requeue the item so we can - // attempt processing again later. This could have been caused by a - // temporary network failure, or any other transient reason. - if err != nil { - return err - } - - if !metav1.IsControlledBy(serviceaccount, submarine) { - msg := fmt.Sprintf(MessageResourceExists, serviceaccount.Name) - c.recorder.Event(submarine, corev1.EventTypeWarning, ErrResourceExists, msg) - return fmt.Errorf(msg) - } - - // Step2: Create Service + // Step1: Create Service service, err := c.serviceLister.Services(submarine.Namespace).Get(serverName) // If the resource doesn't exist, we'll create it if errors.IsNotFound(err) { @@ -153,7 +119,7 @@ func (c *Controller) createSubmarineServer(submarine *v1alpha1.Submarine) error return fmt.Errorf(msg) } - // Step3: Create Deployment + // Step2: Create Deployment deployment, err := c.deploymentLister.Deployments(submarine.Namespace).Get(serverName) // If the resource doesn't exist, we'll create it if errors.IsNotFound(err) { diff --git a/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go b/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go index 61e0bb81..12268a26 100644 --- a/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go +++ b/submarine-cloud-v2/pkg/controller/submarine_server_rbac.go @@ -30,7 +30,20 @@ import ( "k8s.io/klog/v2" ) -func newSubmarineServerRole(submarine *v1alpha1.Submarine) *rbacv1.Role { +func newSubmarineServerServiceAccount(submarine *v1alpha1.Submarine) *corev1.ServiceAccount { + serviceAccount, err := ParseServiceAccountYaml(serverYamlPath) + if err != nil { + klog.Info("[Error] ParseServiceAccountYaml", err) + } + + serviceAccount.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ + *metav1.NewControllerRef(submarine, v1alpha1.SchemeGroupVersion.WithKind("Submarine")), + } + + return serviceAccount +} + +func newSubmarineServerRole(c *Controller, submarine *v1alpha1.Submarine) *rbacv1.Role { role, err := ParseRoleYaml(rbacYamlPath) if err != nil { klog.Info("[Error] ParseRole", err) @@ -39,6 +52,15 @@ func newSubmarineServerRole(submarine *v1alpha1.Submarine) *rbacv1.Role { *metav1.NewControllerRef(submarine, v1alpha1.SchemeGroupVersion.WithKind("Submarine")), } + if c.createPodSecurityPolicy { + // If cluster type is openshift and need create pod security policy, we need add anyuid scc, or we add k8s psp + if c.clusterType == "openshift" { + role.Rules = append(role.Rules, openshiftAnyuidRoleRule) + } else { + role.Rules = append(role.Rules, k8sAnyuidRoleRule) + } + } + return role } @@ -59,11 +81,32 @@ func newSubmarineServerRoleBinding(submarine *v1alpha1.Submarine) *rbacv1.RoleBi func (c *Controller) createSubmarineServerRBAC(submarine *v1alpha1.Submarine) error { klog.Info("[createSubmarineServerRBAC]") - // Step1: Create Role + // Step1: Create ServiceAccount + serviceaccount, err := c.serviceaccountLister.ServiceAccounts(submarine.Namespace).Get(serverName) + // If the resource doesn't exist, we'll create it + if errors.IsNotFound(err) { + serviceaccount, err = c.kubeclientset.CoreV1().ServiceAccounts(submarine.Namespace).Create(context.TODO(), newSubmarineServerServiceAccount(submarine), metav1.CreateOptions{}) + klog.Info(" Create ServiceAccount: ", serviceaccount.Name) + } + + // If an error occurs during Get/Create, we'll requeue the item so we can + // attempt processing again later. This could have been caused by a + // temporary network failure, or any other transient reason. + if err != nil { + return err + } + + if !metav1.IsControlledBy(serviceaccount, submarine) { + msg := fmt.Sprintf(MessageResourceExists, serviceaccount.Name) + c.recorder.Event(submarine, corev1.EventTypeWarning, ErrResourceExists, msg) + return fmt.Errorf(msg) + } + + // Step2: Create Role role, err := c.roleLister.Roles(submarine.Namespace).Get(serverName) // If the resource doesn't exist, we'll create it if errors.IsNotFound(err) { - role, err = c.kubeclientset.RbacV1().Roles(submarine.Namespace).Create(context.TODO(), newSubmarineServerRole(submarine), metav1.CreateOptions{}) + role, err = c.kubeclientset.RbacV1().Roles(submarine.Namespace).Create(context.TODO(), newSubmarineServerRole(c, submarine), metav1.CreateOptions{}) klog.Info(" Create Role: ", role.Name) } @@ -80,6 +123,7 @@ func (c *Controller) createSubmarineServerRBAC(submarine *v1alpha1.Submarine) er return fmt.Errorf(msg) } + // Step3: Create Role Binding rolebinding, rolebinding_err := c.rolebindingLister.RoleBindings(submarine.Namespace).Get(serverName) // If the resource doesn't exist, we'll create it if errors.IsNotFound(rolebinding_err) { diff --git a/submarine-cloud-v2/pkg/controller/submarine_storage_rbac.go b/submarine-cloud-v2/pkg/controller/submarine_storage_rbac.go new file mode 100644 index 00000000..05a38adf --- /dev/null +++ b/submarine-cloud-v2/pkg/controller/submarine_storage_rbac.go @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "fmt" + v1alpha1 "github.com/apache/submarine/submarine-cloud-v2/pkg/apis/submarine/v1alpha1" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" +) + +func newSubmarineStorageRole(c *Controller, submarine *v1alpha1.Submarine) *rbacv1.Role { + role, err := ParseRoleYaml(storageRbacYamlPath) + if err != nil { + klog.Info("[Error] ParseRole", err) + } + role.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ + *metav1.NewControllerRef(submarine, v1alpha1.SchemeGroupVersion.WithKind("Submarine")), + } + + // If cluster type is openshift and need create pod security policy, we need add anyuid scc, or we add k8s psp + if c.clusterType == "openshift" { + role.Rules = append(role.Rules, openshiftAnyuidRoleRule) + } else { + role.Rules = append(role.Rules, k8sAnyuidRoleRule) + } + + return role +} + +func newSubmarineStorageRoleBinding(submarine *v1alpha1.Submarine) *rbacv1.RoleBinding { + roleBinding, err := ParseRoleBindingYaml(storageRbacYamlPath) + if err != nil { + klog.Info("[Error] ParseRoleBinding", err) + } + roleBinding.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ + *metav1.NewControllerRef(submarine, v1alpha1.SchemeGroupVersion.WithKind("Submarine")), + } + + return roleBinding +} + +func newSubmarineStorageServiceAccount(submarine *v1alpha1.Submarine) *corev1.ServiceAccount { + serviceAccount, err := ParseServiceAccountYaml(storageRbacYamlPath) + if err != nil { + klog.Info("[Error] ParseServiceAccountYaml", err) + } + + serviceAccount.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ + *metav1.NewControllerRef(submarine, v1alpha1.SchemeGroupVersion.WithKind("Submarine")), + } + + return serviceAccount +} + +// createSubmarineStorageRBAC is a function to create RBAC for submarine-database and submarine-minio which will be binded on service account: submarine-storage. +// Reference: https://github.com/apache/submarine/blob/master/submarine-cloud-v2/artifacts/submarine/submarine-storage-rbac.yaml +func (c *Controller) createSubmarineStorageRBAC(submarine *v1alpha1.Submarine) error { + klog.Info("[createSubmarineStorageRBAC]") + + // Step1: Create ServiceAccount + serviceaccount, err := c.serviceaccountLister.ServiceAccounts(submarine.Namespace).Get(storageName) + // If the resource doesn't exist, we'll create it + if errors.IsNotFound(err) { + serviceaccount, err = c.kubeclientset.CoreV1().ServiceAccounts(submarine.Namespace).Create(context.TODO(), newSubmarineStorageServiceAccount(submarine), metav1.CreateOptions{}) + klog.Info(" Create ServiceAccount: ", serviceaccount.Name) + } + + // Step2: Pod Security Policy if needed + if c.createPodSecurityPolicy { + // Step2.1: Create Role + role, err := c.roleLister.Roles(submarine.Namespace).Get(storageName) + // If the resource doesn't exist, we'll create it + if errors.IsNotFound(err) { + role, err = c.kubeclientset.RbacV1().Roles(submarine.Namespace).Create(context.TODO(), newSubmarineStorageRole(c, submarine), metav1.CreateOptions{}) + klog.Info(" Create Role: ", role.Name) + } + + // If an error occurs during Get/Create, we'll requeue the item so we can + // attempt processing again later. This could have been caused by a + // temporary network failure, or any other transient reason. + if err != nil { + return err + } + + if !metav1.IsControlledBy(role, submarine) { + msg := fmt.Sprintf(MessageResourceExists, role.Name) + c.recorder.Event(submarine, corev1.EventTypeWarning, ErrResourceExists, msg) + return fmt.Errorf(msg) + } + + // Step2.2: Create Role Binding + rolebinding, rolebinding_err := c.rolebindingLister.RoleBindings(submarine.Namespace).Get(storageName) + // If the resource doesn't exist, we'll create it + if errors.IsNotFound(rolebinding_err) { + rolebinding, rolebinding_err = c.kubeclientset.RbacV1().RoleBindings(submarine.Namespace).Create(context.TODO(), newSubmarineStorageRoleBinding(submarine), metav1.CreateOptions{}) + klog.Info(" Create RoleBinding: ", rolebinding.Name) + } + + // If an error occurs during Get/Create, we'll requeue the item so we can + // attempt processing again later. This could have been caused by a + // temporary network failure, or any other transient reason. + if rolebinding_err != nil { + return rolebinding_err + } + + if !metav1.IsControlledBy(rolebinding, submarine) { + msg := fmt.Sprintf(MessageResourceExists, rolebinding.Name) + c.recorder.Event(submarine, corev1.EventTypeWarning, ErrResourceExists, msg) + return fmt.Errorf(msg) + } + } + + return nil +} --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@submarine.apache.org For additional commands, e-mail: dev-h...@submarine.apache.org