This is an automated email from the ASF dual-hosted git repository.
lidongdai pushed a commit to branch 1.3.5-prepare
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/1.3.5-prepare by this push:
new 263e94d [1.3.5-prepare][Improvement][K8s] Support resource storage
when fs.defaultFS=file:/// (#4700)
263e94d is described below
commit 263e94d177da6cc48b3350cca35d858a023d697d
Author: Shiwen Cheng <[email protected]>
AuthorDate: Sat Feb 6 12:02:40 2021 +0800
[1.3.5-prepare][Improvement][K8s] Support resource storage when
fs.defaultFS=file:/// (#4700)
---
docker/kubernetes/dolphinscheduler/README.md | 4 +++
.../templates/deployment-dolphinscheduler-api.yaml | 11 ++++++-
.../templates/pvc-dolphinscheduler-resource.yaml | 37 ++++++++++++++++++++++
.../statefulset-dolphinscheduler-master.yaml | 2 +-
.../statefulset-dolphinscheduler-worker.yaml | 11 ++++++-
docker/kubernetes/dolphinscheduler/values.yaml | 10 +++++-
6 files changed, 71 insertions(+), 4 deletions(-)
diff --git a/docker/kubernetes/dolphinscheduler/README.md
b/docker/kubernetes/dolphinscheduler/README.md
index cf69346..c4e6fab 100644
--- a/docker/kubernetes/dolphinscheduler/README.md
+++ b/docker/kubernetes/dolphinscheduler/README.md
@@ -80,6 +80,10 @@ The following tables lists the configurable parameters of
the Dolphins Scheduler
| `resource.fsS3aEndpoint`
| S3 endpoint when `resource.resourceStorageType` be seted to `S3`
| `s3.xxx.amazonaws.com`
|
| `resource.fsS3aAccessKey`
| S3 access key when `resource.resourceStorageType` be seted to `S3`
| `xxxxxxx`
|
| `resource.fsS3aSecretKey`
| S3 secret key when `resource.resourceStorageType` be seted to `S3`
| `xxxxxxx`
|
+| `resource.persistentVolumeClaim.enabled`
| Set `resource.persistentVolumeClaim.enabled` to `true` to mount a new
resource volume for `api` and `worker` | `false`
|
+| `resource.persistentVolumeClaim.accessModes`
| `PersistentVolumeClaim` Access Modes, must be `ReadWriteMany`
| `[ReadWriteMany]`
|
+| `resource.persistentVolumeClaim.storageClassName`
| Resource Persistent Volume Storage Class, must support the access mode:
ReadWriteMany | `-`
|
+| `resource.persistentVolumeClaim.storage`
| `PersistentVolumeClaim` Size
| `20Gi`
|
|
|
|
|
| `master.podManagementPolicy`
| PodManagementPolicy controls how pods are created during initial scale
up, when replacing pods on nodes, or when scaling down | `Parallel`
|
| `master.replicas`
| Replicas is the desired number of replicas of the given Template
| `3`
|
diff --git
a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
index 0db3d55..83d9981 100644
---
a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
+++
b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
@@ -166,6 +166,10 @@ spec:
volumeMounts:
- mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-api
+ {{- if .Values.resource.persistentVolumeClaim.enabled }}
+ - mountPath: {{ .Values.resource.resourceUploadPath }}
+ name: {{ include "dolphinscheduler.fullname" . }}-resource
+ {{- end }}
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-api
{{- if .Values.api.persistentVolumeClaim.enabled }}
@@ -173,4 +177,9 @@ spec:
claimName: {{ include "dolphinscheduler.fullname" . }}-api
{{- else }}
emptyDir: {}
- {{- end }}
\ No newline at end of file
+ {{- end }}
+ {{- if .Values.resource.persistentVolumeClaim.enabled }}
+ - name: {{ include "dolphinscheduler.fullname" . }}-resource
+ persistentVolumeClaim:
+ claimName: {{ include "dolphinscheduler.fullname" . }}-resource
+ {{- end }}
\ No newline at end of file
diff --git
a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-resource.yaml
b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-resource.yaml
new file mode 100644
index 0000000..3494800
--- /dev/null
+++
b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-resource.yaml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if .Values.resource.persistentVolumeClaim.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: {{ include "dolphinscheduler.fullname" . }}-resource
+ labels:
+ app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" .
}}-resource
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/managed-by: {{ .Release.Service }}
+ annotations:
+ "helm.sh/resource-policy": keep
+spec:
+ accessModes:
+ {{- range .Values.resource.persistentVolumeClaim.accessModes }}
+ - {{ . | quote }}
+ {{- end }}
+ storageClassName: {{ .Values.resource.persistentVolumeClaim.storageClassName
| quote }}
+ resources:
+ requests:
+ storage: {{ .Values.resource.persistentVolumeClaim.storage | quote }}
+{{- end }}
\ No newline at end of file
diff --git
a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
index 7f42205..839cef5 100644
---
a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
+++
b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
@@ -202,4 +202,4 @@ spec:
resources:
requests:
storage: {{ .Values.master.persistentVolumeClaim.storage | quote }}
- {{- end }}
+ {{- end }}
\ No newline at end of file
diff --git
a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
index a390a7b..4e692da 100644
---
a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
+++
b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
@@ -211,6 +211,10 @@ spec:
- mountPath:
"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"
subPath: "dolphinscheduler_env.sh"
name: {{ include "dolphinscheduler.fullname" .
}}-worker-configmap
+ {{- if .Values.resource.persistentVolumeClaim.enabled }}
+ - mountPath: {{ .Values.resource.resourceUploadPath }}
+ name: {{ include "dolphinscheduler.fullname" . }}-resource
+ {{- end }}
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-worker-data
{{- if
.Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
@@ -233,6 +237,11 @@ spec:
items:
- key: dolphinscheduler_env.sh
path: dolphinscheduler_env.sh
+ {{- if .Values.resource.persistentVolumeClaim.enabled }}
+ - name: {{ include "dolphinscheduler.fullname" . }}-resource
+ persistentVolumeClaim:
+ claimName: {{ include "dolphinscheduler.fullname" . }}-resource
+ {{- end }}
{{- if .Values.worker.persistentVolumeClaim.enabled }}
volumeClaimTemplates:
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
@@ -269,4 +278,4 @@ spec:
requests:
storage: {{
.Values.worker.persistentVolumeClaim.logsPersistentVolume.storage | quote }}
{{- end }}
- {{- end }}
+ {{- end }}
\ No newline at end of file
diff --git a/docker/kubernetes/dolphinscheduler/values.yaml
b/docker/kubernetes/dolphinscheduler/values.yaml
index 5206450..d154c87 100644
--- a/docker/kubernetes/dolphinscheduler/values.yaml
+++ b/docker/kubernetes/dolphinscheduler/values.yaml
@@ -65,7 +65,7 @@ zookeeper:
zookeeperRoot: /dolphinscheduler
# If exists external zookeeper, and set zookeeper.enable value to false.
-# If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it.
+# If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it.
externalZookeeper:
zookeeperQuorum: "127.0.0.1:2181"
zookeeperRoot: /dolphinscheduler
@@ -77,6 +77,14 @@ resource:
fsS3aEndpoint: "s3.xxx.amazonaws.com"
fsS3aAccessKey: "xxxxxxx"
fsS3aSecretKey: "xxxxxxx"
+ # If resourceStorageType is HDFS and fsDefaultFS is file:///,
persistentVolumeClaim should be enabled for resource storage
+ persistentVolumeClaim:
+ enabled: false
+ accessModes:
+ - "ReadWriteMany"
+ # storageClassName must support the access mode: ReadWriteMany
+ storageClassName: "-"
+ storage: "20Gi"
master:
podManagementPolicy: "Parallel"
