This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new af99d5a161 Add volumeClaimTemplate for worker in Helm chart (#34986)
af99d5a161 is described below
commit af99d5a16177c867f9d848302260c43bf929bf44
Author: KHTee <75174583+kh...@users.noreply.github.com>
AuthorDate: Mon Oct 30 14:24:05 2023 +0800
Add volumeClaimTemplate for worker in Helm chart (#34986)
* Add volumeClaimTemplate for worker in Helm chart
---------
Co-authored-by: Isaac <kahhui....@cognixus.com>
Co-authored-by: Hussein Awala <huss...@awala.fr>
---
chart/templates/workers/worker-deployment.yaml | 3 ++
chart/values.schema.json | 34 ++++++++++++++++++
chart/values.yaml | 25 +++++++++++++
helm_tests/airflow_core/test_worker.py | 49 ++++++++++++++++++++++++++
4 files changed, 111 insertions(+)
diff --git a/chart/templates/workers/worker-deployment.yaml
b/chart/templates/workers/worker-deployment.yaml
index 5d1adbb465..a1fdfc72c4 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -398,5 +398,8 @@ spec:
resources:
requests:
storage: {{ .Values.workers.persistence.size }}
+ {{- with .Values.workers.volumeClaimTemplates }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
{{- end }}
{{- end }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 711cbf1a16..6f84338b4a 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -2020,6 +2020,40 @@
],
"additionalProperties": false
}
+ },
+ "volumeClaimTemplates": {
+ "description": "Specify additional volume claim template
for workers.",
+ "type": "array",
+ "default": [],
+ "items": {
+ "$ref":
"#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimTemplate"
+ },
+ "examples": [
+ {
+ "name": "data-volume-1",
+ "storageClassName": "storage-class-1",
+ "accessModes": [
+ "ReadWriteOnce"
+ ],
+ "resources": {
+ "requests": {
+ "storage": "10Gi"
+ }
+ }
+ },
+ {
+ "name": "data-volume-2",
+ "storageClassName": "storage-class-2",
+ "accessModes": [
+ "ReadWriteOnce"
+ ],
+ "resources": {
+ "requests": {
+ "storage": "20Gi"
+ }
+ }
+ }
+ ]
}
}
},
diff --git a/chart/values.yaml b/chart/values.yaml
index 859f25e020..d16649118b 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -729,6 +729,31 @@ workers:
env: []
+ volumeClaimTemplates: []
+ # Additional volumeClaimTemplates needed.
+ # Comment out the above and uncomment the section below to enable it.
+ # Add more as needed
+ # Make sure to mount it under extraVolumeMounts.
+ # volumeClaimTemplates:
+ # - metadata:
+ # name: data-volume-1
+ # spec:
+ # storageClassName: "storage-class-1"
+ # accessModes:
+ # - "ReadWriteOnce"
+ # resources:
+ # requests:
+ # storage: "10Gi"
+ # - metadata:
+ # name: data-volume-2
+ # spec:
+ # storageClassName: "storage-class-2"
+ # accessModes:
+ # - "ReadWriteOnce"
+ # resources:
+ # requests:
+ # storage: "20Gi"
+
# Airflow scheduler settings
scheduler:
# hostAliases for the scheduler pod
diff --git a/helm_tests/airflow_core/test_worker.py
b/helm_tests/airflow_core/test_worker.py
index a04a42e2ad..3eae2d413a 100644
--- a/helm_tests/airflow_core/test_worker.py
+++ b/helm_tests/airflow_core/test_worker.py
@@ -667,6 +667,55 @@ class TestWorker:
== evictionStr
)
+ def test_should_add_extra_volume_claim_templates(self):
+ docs = render_chart(
+ values={
+ "executor": "CeleryExecutor",
+ "workers": {
+ "volumeClaimTemplates": [
+ {
+ "metadata": {"name": "test-volume-airflow-1"},
+ "spec": {
+ "storageClassName": "storage-class-1",
+ "accessModes": ["ReadWriteOnce"],
+ "resources": {"requests": {"storage": "10Gi"}},
+ },
+ },
+ {
+ "metadata": {"name": "test-volume-airflow-2"},
+ "spec": {
+ "storageClassName": "storage-class-2",
+ "accessModes": ["ReadWriteOnce"],
+ "resources": {"requests": {"storage": "20Gi"}},
+ },
+ },
+ ]
+ },
+ },
+ show_only=["templates/workers/worker-deployment.yaml"],
+ )
+
+ assert "test-volume-airflow-1" == jmespath.search(
+ "spec.volumeClaimTemplates[1].metadata.name", docs[0]
+ )
+ assert "test-volume-airflow-2" == jmespath.search(
+ "spec.volumeClaimTemplates[2].metadata.name", docs[0]
+ )
+ assert "storage-class-1" == jmespath.search(
+ "spec.volumeClaimTemplates[1].spec.storageClassName", docs[0]
+ )
+ assert "storage-class-2" == jmespath.search(
+ "spec.volumeClaimTemplates[2].spec.storageClassName", docs[0]
+ )
+ assert ["ReadWriteOnce"] ==
jmespath.search("spec.volumeClaimTemplates[1].spec.accessModes", docs[0])
+ assert ["ReadWriteOnce"] ==
jmespath.search("spec.volumeClaimTemplates[2].spec.accessModes", docs[0])
+ assert "10Gi" == jmespath.search(
+ "spec.volumeClaimTemplates[1].spec.resources.requests.storage",
docs[0]
+ )
+ assert "20Gi" == jmespath.search(
+ "spec.volumeClaimTemplates[2].spec.resources.requests.storage",
docs[0]
+ )
+
class TestWorkerLogGroomer(LogGroomerTestBase):
"""Worker groomer."""
