This is an automated email from the ASF dual-hosted git repository. mmarshall pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar-helm-chart.git
The following commit(s) were added to refs/heads/master by this push: new 91f8b6f Add multi volume support in bookkeeper. (#113) 91f8b6f is described below commit 91f8b6f6b19d6da519694aaaa48c8c9cfc075055 Author: Shen Liu <liushen...@126.com> AuthorDate: Sun Jan 23 13:08:07 2022 +0800 Add multi volume support in bookkeeper. (#113) * Add multi volume support in bookkeeper. (#112) * Add multi volumes support in bookkeeper configmap. Co-authored-by: druidliu <druid...@tencent.com> Fixes #112 ### Motivation *Add option for user to choose whether using multi volume in bookeeper, especially while using `local-storage`.* ### Modifications Add `useMultiVolumes` option under `.Values.bookkeeper.volumes.journal` and `.Values.bookkeeper.volumes.ledgers`. User can choose how many volumes could be used for bookkeeper jounal or ledgers. ### Verifying this change - [x] Make sure that the change passes the CI checks. --- charts/pulsar/templates/bookkeeper-configmap.yaml | 17 +++++++ .../pulsar/templates/bookkeeper-statefulset.yaml | 53 +++++++++++++++++++++- charts/pulsar/values.yaml | 20 ++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/charts/pulsar/templates/bookkeeper-configmap.yaml b/charts/pulsar/templates/bookkeeper-configmap.yaml index a99ee20..8ed5f9b 100644 --- a/charts/pulsar/templates/bookkeeper-configmap.yaml +++ b/charts/pulsar/templates/bookkeeper-configmap.yaml @@ -35,9 +35,26 @@ data: {{- end }} # Do not retain journal files as it increase the disk utilization journalMaxBackups: "0" + {{- if .Values.bookkeeper.volumes.journal.useMultiVolumes }} + {{- $journalDirs := list -}} + {{ range .Values.bookkeeper.volumes.journal.multiVolumes }} + {{- $journalDirs = append $journalDirs .mountPath -}} + {{- end }} + journalDirectories: {{ $journalDirs | join "," | quote }} + PULSAR_PREFIX_journalDirectories: {{ $journalDirs | join "," | quote }} + {{- else }} journalDirectories: "/pulsar/data/bookkeeper/journal" PULSAR_PREFIX_journalDirectories: "/pulsar/data/bookkeeper/journal" + {{- end }} + {{- if .Values.bookkeeper.volumes.ledgers.useMultiVolumes }} + {{- $ledgerDirs := list -}} + {{ range .Values.bookkeeper.volumes.ledgers.multiVolumes }} + {{- $ledgerDirs = append $ledgerDirs .mountPath -}} + {{- end }} + ledgerDirectories: {{ $ledgerDirs | join "," | quote }} + {{- else }} ledgerDirectories: "/pulsar/data/bookkeeper/ledgers" + {{- end }} # TLS config {{- include "pulsar.bookkeeper.config.tls" . | nindent 2 }} {{ toYaml .Values.bookkeeper.configData | indent 2 }} diff --git a/charts/pulsar/templates/bookkeeper-statefulset.yaml b/charts/pulsar/templates/bookkeeper-statefulset.yaml index 985f00b..80473f9 100644 --- a/charts/pulsar/templates/bookkeeper-statefulset.yaml +++ b/charts/pulsar/templates/bookkeeper-statefulset.yaml @@ -188,11 +188,28 @@ spec: - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.common.name }}" mountPath: /pulsar/data/bookkeeper {{- else }} + {{- if .Values.bookkeeper.volumes.journal.useMultiVolumes }} + {{- $fullname := include "pulsar.fullname" . -}} + {{- $bkComponent := .Values.bookkeeper.component -}} + {{ range .Values.bookkeeper.volumes.journal.multiVolumes }} + - name: "{{ $fullname }}-{{ $bkComponent }}-{{ .name }}" + mountPath: {{ .mountPath }} + {{- end }} + {{- else }} - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.journal.name }}" mountPath: /pulsar/data/bookkeeper/journal + {{- end}} + {{- if .Values.bookkeeper.volumes.ledgers.useMultiVolumes }} + {{- $fullname := include "pulsar.fullname" . -}} + {{- $bkComponent := .Values.bookkeeper.component -}} + {{ range .Values.bookkeeper.volumes.ledgers.multiVolumes }} + - name: "{{ $fullname }}-{{ $bkComponent }}-{{ .name }}" + mountPath: {{ .mountPath }} + {{- end }} + {{- else }} - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}" mountPath: /pulsar/data/bookkeeper/ledgers - {{- end}} + {{- end }} {{- if .Values.bookkeeper.extraVolumeMounts }} {{ toYaml .Values.bookkeeper.extraVolumeMounts | indent 8 }} {{- end }} @@ -227,6 +244,22 @@ spec: storageClassName: "local-storage" {{- end }} {{- else }} + {{- if .Values.bookkeeper.volumes.journal.useMultiVolumes }} + {{- $fullname := include "pulsar.fullname" . -}} + {{- $bkComponent := .Values.bookkeeper.component -}} + {{ range .Values.bookkeeper.volumes.journal.multiVolumes }} + - metadata: + name: "{{ $fullname }}-{{ $bkComponent }}-{{ .name }}" + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .size }} + {{- if .storageClassName }} + storageClassName: "{{ .storageClassName }}" + {{- end }} + {{- end }} + {{- else }} - metadata: name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.journal.name }}" spec: @@ -241,6 +274,23 @@ spec: {{- else if and .Values.volumes.local_storage .Values.bookkeeper.volumes.journal.local_storage }} storageClassName: "local-storage" {{- end }} + {{- end }} + {{- if .Values.bookkeeper.volumes.ledgers.useMultiVolumes }} + {{- $fullname := include "pulsar.fullname" . -}} + {{- $bkComponent := .Values.bookkeeper.component -}} + {{ range .Values.bookkeeper.volumes.ledgers.multiVolumes }} + - metadata: + name: "{{ $fullname }}-{{ $bkComponent }}-{{ .name }}" + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .size }} + {{- if .storageClassName }} + storageClassName: "{{ .storageClassName }}" + {{- end }} + {{- end }} + {{- else }} - metadata: name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}" spec: @@ -256,5 +306,6 @@ spec: storageClassName: "local-storage" {{- end }} {{- end }} + {{- end }} {{- end }} {{- end }} diff --git a/charts/pulsar/values.yaml b/charts/pulsar/values.yaml index b162c11..239a2b5 100644 --- a/charts/pulsar/values.yaml +++ b/charts/pulsar/values.yaml @@ -506,6 +506,16 @@ bookkeeper: # type: pd-ssd # fsType: xfs # provisioner: kubernetes.io/gce-pd + useMultiVolumes: false + multiVolumes: + - name: journal0 + size: 10Gi + # storageClassName: existent-storage-class + mountPath: /pulsar/data/bookkeeper/journal0 + - name: journal1 + size: 10Gi + # storageClassName: existent-storage-class + mountPath: /pulsar/data/bookkeeper/journal1 ledgers: name: ledgers size: 50Gi @@ -513,6 +523,16 @@ bookkeeper: # storageClassName: # storageClass: # ... + useMultiVolumes: false + multiVolumes: + - name: ledgers0 + size: 10Gi + # storageClassName: existent-storage-class + mountPath: /pulsar/data/bookkeeper/ledgers0 + - name: ledgers1 + size: 10Gi + # storageClassName: existent-storage-class + mountPath: /pulsar/data/bookkeeper/ledgers1 ## use a single common volume for both journal and ledgers useSingleCommonVolume: false