This is an automated email from the ASF dual-hosted git repository. eladkal pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new dc74d3327e Feature/add templating for PVC storageClassName (#35581) dc74d3327e is described below commit dc74d3327e6aed105b7aa7f08a79ee6341543747 Author: rom sharon <33751805+romsharo...@users.noreply.github.com> AuthorDate: Sun Nov 12 16:46:27 2023 +0200 Feature/add templating for PVC storageClassName (#35581) * add tests * rebasee * rebase * fix(helm): safer worker pod annotations (#35309) * chore(helm): safer worker pod annotations * chore(helm): safer worker pod annotations --------- Co-authored-by: hakuno <m...@seiji.com.br> Co-authored-by: Hussein Awala <huss...@awala.fr> * add tests * change from rebase * fix statics * fix tests * fix tests * check * check * fix_log_test * fix statics * run precommits --------- Co-authored-by: Seiji 誠 次 <seose...@gmail.com> Co-authored-by: hakuno <m...@seiji.com.br> Co-authored-by: Hussein Awala <huss...@awala.fr> --- chart/templates/dags-persistent-volume-claim.yaml | 2 +- chart/templates/logs-persistent-volume-claim.yaml | 2 +- chart/templates/redis/redis-statefulset.yaml | 2 +- chart/templates/scheduler/scheduler-deployment.yaml | 2 +- chart/templates/triggerer/triggerer-deployment.yaml | 2 +- chart/templates/workers/worker-deployment.yaml | 2 +- .../airflow_aux/test_logs_persistent_volume_claim.py | 15 +++++++++++++++ helm_tests/airflow_core/test_scheduler.py | 18 ++++++++++++++++++ helm_tests/airflow_core/test_triggerer.py | 9 +++++++++ helm_tests/airflow_core/test_worker.py | 9 +++++++++ helm_tests/other/test_dags_persistent_volume_claim.py | 15 +++++++++++++++ helm_tests/other/test_redis.py | 9 +++++++++ 12 files changed, 81 insertions(+), 6 deletions(-) diff --git a/chart/templates/dags-persistent-volume-claim.yaml b/chart/templates/dags-persistent-volume-claim.yaml index 15628cac46..c0999aa0b4 100644 --- a/chart/templates/dags-persistent-volume-claim.yaml +++ b/chart/templates/dags-persistent-volume-claim.yaml @@ -46,7 +46,7 @@ spec: {{- if (eq "-" .Values.dags.persistence.storageClassName) }} storageClassName: "" {{- else }} - storageClassName: "{{ .Values.dags.persistence.storageClassName }}" + storageClassName: {{ tpl .Values.dags.persistence.storageClassName . | quote }} {{- end }} {{- end }} {{- end }} diff --git a/chart/templates/logs-persistent-volume-claim.yaml b/chart/templates/logs-persistent-volume-claim.yaml index a45088858b..aa5b11dada 100644 --- a/chart/templates/logs-persistent-volume-claim.yaml +++ b/chart/templates/logs-persistent-volume-claim.yaml @@ -46,7 +46,7 @@ spec: {{- if (eq "-" .Values.logs.persistence.storageClassName) }} storageClassName: "" {{- else }} - storageClassName: "{{ .Values.logs.persistence.storageClassName }}" + storageClassName: {{ tpl .Values.logs.persistence.storageClassName . | quote }} {{- end }} {{- end }} {{- end }} diff --git a/chart/templates/redis/redis-statefulset.yaml b/chart/templates/redis/redis-statefulset.yaml index b978373b94..5aa9e13598 100644 --- a/chart/templates/redis/redis-statefulset.yaml +++ b/chart/templates/redis/redis-statefulset.yaml @@ -119,7 +119,7 @@ spec: {{- end }} spec: {{- if .Values.redis.persistence.storageClassName }} - storageClassName: {{ .Values.redis.persistence.storageClassName }} + storageClassName: {{ tpl .Values.redis.persistence.storageClassName . | quote }} {{- end }} accessModes: ["ReadWriteOnce"] resources: diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 8e599b4a28..ccee1609fb 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -326,7 +326,7 @@ spec: {{- end }} spec: {{- if .Values.workers.persistence.storageClassName }} - storageClassName: {{ .Values.workers.persistence.storageClassName }} + storageClassName: {{ tpl .Values.workers.persistence.storageClassName . | quote }} {{- end }} accessModes: ["ReadWriteOnce"] resources: diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index ef6f66db85..6201a53494 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -297,7 +297,7 @@ spec: {{- end }} spec: {{- if .Values.triggerer.persistence.storageClassName }} - storageClassName: {{ .Values.triggerer.persistence.storageClassName }} + storageClassName: {{ tpl .Values.triggerer.persistence.storageClassName . | quote }} {{- end }} accessModes: ["ReadWriteOnce"] resources: diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index b4dcb4cb96..33fb357aae 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -429,7 +429,7 @@ spec: {{- end }} spec: {{- if .Values.workers.persistence.storageClassName }} - storageClassName: {{ .Values.workers.persistence.storageClassName }} + storageClassName: {{ tpl .Values.workers.persistence.storageClassName . | quote }} {{- end }} accessModes: ["ReadWriteOnce"] resources: diff --git a/helm_tests/airflow_aux/test_logs_persistent_volume_claim.py b/helm_tests/airflow_aux/test_logs_persistent_volume_claim.py index f815b3d217..c3217ee741 100644 --- a/helm_tests/airflow_aux/test_logs_persistent_volume_claim.py +++ b/helm_tests/airflow_aux/test_logs_persistent_volume_claim.py @@ -68,3 +68,18 @@ class TestLogsPersistentVolumeClaim: "resources": {"requests": {"storage": "1G"}}, "storageClassName": "MyStorageClass", } == jmespath.search("spec", docs[0]) + + def test_logs_persistent_volume_claim_template_storage_class_name(self): + docs = render_chart( + values={ + "logs": { + "persistence": { + "existingClaim": None, + "enabled": True, + "storageClassName": "{{ .Release.Name }}-storage-class", + } + } + }, + show_only=["templates/logs-persistent-volume-claim.yaml"], + ) + assert "release-name-storage-class" == jmespath.search("spec.storageClassName", docs[0]) diff --git a/helm_tests/airflow_core/test_scheduler.py b/helm_tests/airflow_core/test_scheduler.py index 1146151016..addaaf896e 100644 --- a/helm_tests/airflow_core/test_scheduler.py +++ b/helm_tests/airflow_core/test_scheduler.py @@ -774,6 +774,24 @@ class TestScheduler: assert "127.0.0.1" == jmespath.search("spec.template.spec.hostAliases[0].ip", docs[0]) assert "foo.local" == jmespath.search("spec.template.spec.hostAliases[0].hostnames[0]", docs[0]) + def test_scheduler_template_storage_class_name(self): + docs = render_chart( + values={ + "workers": { + "persistence": { + "storageClassName": "{{ .Release.Name }}-storage-class", + "enabled": True, + } + }, + "logs": {"persistence": {"enabled": False}}, + "executor": "LocalExecutor", + }, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + ) + assert "release-name-storage-class" == jmespath.search( + "spec.volumeClaimTemplates[0].spec.storageClassName", docs[0] + ) + class TestSchedulerNetworkPolicy: """Tests scheduler network policy.""" diff --git a/helm_tests/airflow_core/test_triggerer.py b/helm_tests/airflow_core/test_triggerer.py index 369e874070..4aa1126072 100644 --- a/helm_tests/airflow_core/test_triggerer.py +++ b/helm_tests/airflow_core/test_triggerer.py @@ -592,6 +592,15 @@ class TestTriggerer: assert "annotations" in jmespath.search("metadata", docs[0]) assert jmespath.search("metadata.annotations", docs[0])["test_annotation"] == "test_annotation_value" + def test_triggerer_template_storage_class_name(self): + docs = render_chart( + values={"triggerer": {"persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}}}, + show_only=["templates/triggerer/triggerer-deployment.yaml"], + ) + assert "release-name-storage-class" == jmespath.search( + "spec.volumeClaimTemplates[0].spec.storageClassName", docs[0] + ) + class TestTriggererServiceAccount: """Tests triggerer service account.""" diff --git a/helm_tests/airflow_core/test_worker.py b/helm_tests/airflow_core/test_worker.py index c9dfdb0f53..a65db04a5d 100644 --- a/helm_tests/airflow_core/test_worker.py +++ b/helm_tests/airflow_core/test_worker.py @@ -884,6 +884,15 @@ class TestWorker: else: assert jmespath.search("spec.template.metadata.annotations.scope", docs[0]) is None + def test_worker_template_storage_class_name(self): + docs = render_chart( + values={"workers": {"persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}}}, + show_only=["templates/workers/worker-deployment.yaml"], + ) + assert "release-name-storage-class" == jmespath.search( + "spec.volumeClaimTemplates[0].spec.storageClassName", docs[0] + ) + class TestWorkerLogGroomer(LogGroomerTestBase): """Worker groomer.""" diff --git a/helm_tests/other/test_dags_persistent_volume_claim.py b/helm_tests/other/test_dags_persistent_volume_claim.py index 527a1d859a..8667bcc4c8 100644 --- a/helm_tests/other/test_dags_persistent_volume_claim.py +++ b/helm_tests/other/test_dags_persistent_volume_claim.py @@ -110,3 +110,18 @@ class TestDagsPersistentVolumeClaim: annotations = jmespath.search("metadata.annotations", docs[0]) assert "value" == annotations.get("key") assert "value-two" == annotations.get("key-two") + + def test_dags_persistent_volume_claim_template_storage_class_name(self): + docs = render_chart( + values={ + "dags": { + "persistence": { + "existingClaim": None, + "enabled": True, + "storageClassName": "{{ .Release.Name }}-storage-class", + } + } + }, + show_only=["templates/dags-persistent-volume-claim.yaml"], + ) + assert "release-name-storage-class" == jmespath.search("spec.storageClassName", docs[0]) diff --git a/helm_tests/other/test_redis.py b/helm_tests/other/test_redis.py index cdbb5e2d5a..843bb3d7b1 100644 --- a/helm_tests/other/test_redis.py +++ b/helm_tests/other/test_redis.py @@ -368,6 +368,15 @@ class TestRedis: docs[0], ) + def test_redis_template_storage_class_name(self): + docs = render_chart( + values={"redis": {"persistence": {"storageClassName": "{{ .Release.Name }}-storage-class"}}}, + show_only=["templates/redis/redis-statefulset.yaml"], + ) + assert "release-name-storage-class" == jmespath.search( + "spec.volumeClaimTemplates[0].spec.storageClassName", docs[0] + ) + class TestRedisServiceAccount: """Tests redis service account."""