This is an automated email from the ASF dual-hosted git repository.

jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 03a305f660 Add support for custom emptyDir config (#34837)
03a305f660 is described below

commit 03a305f6600b822d4f4ab74ed5ae8c0f8905fbc2
Author: Pedro Miranda <[email protected]>
AuthorDate: Wed Oct 11 10:24:43 2023 -0500

    Add support for custom emptyDir config (#34837)
---
 chart/files/pod-template-file.kubernetes-helm-yaml |  4 +--
 .../dag-processor/dag-processor-deployment.yaml    |  4 +--
 chart/templates/redis/redis-statefulset.yaml       |  2 +-
 .../templates/scheduler/scheduler-deployment.yaml  |  4 +--
 .../templates/triggerer/triggerer-deployment.yaml  |  4 +--
 .../templates/webserver/webserver-deployment.yaml  |  2 +-
 chart/templates/workers/worker-deployment.yaml     |  4 +--
 chart/values.schema.json                           | 18 ++++++++++++
 chart/values.yaml                                  | 15 ++++++++++
 helm_tests/airflow_aux/test_pod_template_file.py   | 33 +++++++++++++++-------
 helm_tests/airflow_core/test_dag_processor.py      | 19 +++++++++----
 helm_tests/airflow_core/test_scheduler.py          | 19 +++++++++----
 helm_tests/airflow_core/test_triggerer.py          | 19 +++++++++----
 helm_tests/airflow_core/test_worker.py             | 19 +++++++++----
 helm_tests/other/test_redis.py                     | 17 +++++++++++
 15 files changed, 137 insertions(+), 46 deletions(-)

diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml 
b/chart/files/pod-template-file.kubernetes-helm-yaml
index 48e2c476b9..0c35d4ae2f 100644
--- a/chart/files/pod-template-file.kubernetes-helm-yaml
+++ b/chart/files/pod-template-file.kubernetes-helm-yaml
@@ -117,14 +117,14 @@ spec:
       claimName: {{ template "airflow_dags_volume_claim" . }}
   {{- else if .Values.dags.gitSync.enabled }}
   - name: dags
-    emptyDir: {}
+    emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) 
| nindent 6 }}
   {{- end }}
   {{- if .Values.logs.persistence.enabled }}
   - name: logs
     persistentVolumeClaim:
       claimName: {{ template "airflow_logs_volume_claim" . }}
   {{- else }}
-  - emptyDir: {}
+  - emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | 
nindent 6 }}
     name: logs
   {{- end }}
   {{- if and  .Values.dags.gitSync.enabled  .Values.dags.gitSync.sshKeySecret 
}}
diff --git a/chart/templates/dag-processor/dag-processor-deployment.yaml 
b/chart/templates/dag-processor/dag-processor-deployment.yaml
index 4e87bae3c0..61cd3f56bc 100644
--- a/chart/templates/dag-processor/dag-processor-deployment.yaml
+++ b/chart/templates/dag-processor/dag-processor-deployment.yaml
@@ -238,7 +238,7 @@ spec:
             claimName: {{ template "airflow_dags_volume_claim" . }}
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) 
.Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
         {{- end }}
         {{- if and .Values.dags.gitSync.enabled 
.Values.dags.gitSync.sshKeySecret }}
           {{- include "git_sync_ssh_key_volume" . | indent 8 }}
@@ -255,7 +255,7 @@ spec:
             claimName: {{ template "airflow_logs_volume_claim" . }}
         {{- else }}
         - name: logs
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | 
nindent 12 }}
 {{- end }}
 {{- end }}
 {{- end }}
diff --git a/chart/templates/redis/redis-statefulset.yaml 
b/chart/templates/redis/redis-statefulset.yaml
index 224f38f8f5..62220a7ca5 100644
--- a/chart/templates/redis/redis-statefulset.yaml
+++ b/chart/templates/redis/redis-statefulset.yaml
@@ -106,7 +106,7 @@ spec:
   {{- if not .Values.redis.persistence.enabled }}
       volumes:
         - name: redis-db
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.redis.emptyDirConfig) | 
nindent 12 }}
   {{- else }}
   volumeClaimTemplates:
     - metadata:
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml 
b/chart/templates/scheduler/scheduler-deployment.yaml
index 2ea30e3e2e..8e599b4a28 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -298,7 +298,7 @@ spec:
             claimName: {{ template "airflow_dags_volume_claim" . }}
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) 
.Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
         {{- if .Values.dags.gitSync.sshKeySecret }}
           {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
@@ -316,7 +316,7 @@ spec:
             claimName: {{ template "airflow_logs_volume_claim" . }}
   {{- else if not $stateful }}
         - name: logs
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | 
nindent 12 }}
   {{- else }}
   volumeClaimTemplates:
     - metadata:
diff --git a/chart/templates/triggerer/triggerer-deployment.yaml 
b/chart/templates/triggerer/triggerer-deployment.yaml
index bef188f68a..ef6f66db85 100644
--- a/chart/templates/triggerer/triggerer-deployment.yaml
+++ b/chart/templates/triggerer/triggerer-deployment.yaml
@@ -270,7 +270,7 @@ spec:
             claimName: {{ template "airflow_dags_volume_claim" . }}
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) 
.Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
         {{- if .Values.dags.gitSync.sshKeySecret }}
           {{- include "git_sync_ssh_key_volume" . | nindent 8 }}
         {{- end }}
@@ -287,7 +287,7 @@ spec:
             claimName: {{ template "airflow_logs_volume_claim" . }}
   {{- else if not $persistence }}
         - name: logs
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | 
nindent 12 }}
   {{- else }}
   volumeClaimTemplates:
     - metadata:
diff --git a/chart/templates/webserver/webserver-deployment.yaml 
b/chart/templates/webserver/webserver-deployment.yaml
index 26fe76d9ea..1fa06b0724 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -274,7 +274,7 @@ spec:
             claimName: {{ template "airflow_dags_volume_claim" . }}
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) 
.Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
         {{- if  .Values.dags.gitSync.sshKeySecret }}
           {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
diff --git a/chart/templates/workers/worker-deployment.yaml 
b/chart/templates/workers/worker-deployment.yaml
index 213fadbb03..21f7c8fd2d 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -373,7 +373,7 @@ spec:
             claimName: {{ template "airflow_dags_volume_claim" . }}
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) 
.Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
         {{- if  .Values.dags.gitSync.sshKeySecret }}
           {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
@@ -384,7 +384,7 @@ spec:
             claimName: {{ template "airflow_logs_volume_claim" . }}
   {{- else if not $persistence }}
         - name: logs
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | 
nindent 12 }}
   {{- else }}
   volumeClaimTemplates:
     - metadata:
diff --git a/chart/values.schema.json b/chart/values.schema.json
index dce38a205f..8a11d6e672 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -6164,6 +6164,12 @@
                         }
                     }
                 },
+                "emptyDirConfig": {
+                    "description": "Configuration for redis empty dir volume.",
+                    "type": "object",
+                    "$ref": 
"#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource",
+                    "default": null
+                },
                 "resources": {
                     "description": "Resources for the Redis pods",
                     "type": "object",
@@ -7046,6 +7052,12 @@
                                 }
                             ]
                         },
+                        "emptyDirConfig": {
+                            "description": "Configuration for dags empty dir 
volume.",
+                            "type": "object",
+                            "$ref": 
"#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource",
+                            "default": null
+                        },
                         "containerLifecycleHooks": {
                             "description": "Container Lifecycle Hooks 
definition for the git sync sidecar. If not set, the values from global 
`containerLifecycleHooks` will be used.",
                             "type": "object",
@@ -7235,6 +7247,12 @@
                             "default": null
                         }
                     }
+                },
+                "emptyDirConfig": {
+                    "description": "Configuration for logs empty dir volume.",
+                    "type": "object",
+                    "$ref": 
"#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource",
+                    "default": null
                 }
             }
         }
diff --git a/chart/values.yaml b/chart/values.yaml
index 7b238523ab..bc7d19355d 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -2066,6 +2066,11 @@ redis:
     # Annotations to add to redis volumes
     annotations: {}
 
+  # Configuration for empty dir volume (if redis.persistence.enabled == false)
+  # emptyDirConfig:
+  #   sizeLimit: 1Gi
+  #   medium: Memory
+
   resources: {}
   #  limits:
   #   cpu: 100m
@@ -2440,6 +2445,11 @@ dags:
     # - name: ""
     #   value: ""
 
+    # Configuration for empty dir volume
+    # emptyDirConfig:
+    #   sizeLimit: 1Gi
+    #   medium: Memory
+
     resources: {}
     #  limits:
     #   cpu: 100m
@@ -2449,6 +2459,11 @@ dags:
     #   memory: 128Mi
 
 logs:
+  # Configuration for empty dir volume (if logs.persistence.enabled == false)
+  # emptyDirConfig:
+  #   sizeLimit: 1Gi
+  #   medium: Memory
+
   persistence:
     # Enable persistent volume for storing logs
     enabled: false
diff --git a/helm_tests/airflow_aux/test_pod_template_file.py 
b/helm_tests/airflow_aux/test_pod_template_file.py
index e56d77c55a..c1a831e6b1 100644
--- a/helm_tests/airflow_aux/test_pod_template_file.py
+++ b/helm_tests/airflow_aux/test_pod_template_file.py
@@ -267,29 +267,42 @@ class TestPodTemplateFile:
             "spec.volumes", docs[0]
         )
 
-    def test_should_use_empty_dir_for_gitsync_without_persistence(self):
+    @pytest.mark.parametrize(
+        "dags_gitsync_values, expected",
+        [
+            ({"enabled": True}, {"emptyDir": {}}),
+            ({"enabled": True, "emptyDirConfig": {"sizeLimit": "10Gi"}}, 
{"emptyDir": {"sizeLimit": "10Gi"}}),
+        ],
+    )
+    def test_should_use_empty_dir_for_gitsync_without_persistence(self, 
dags_gitsync_values, expected):
         docs = render_chart(
-            values={"dags": {"gitSync": {"enabled": True}}},
+            values={"dags": {"gitSync": dags_gitsync_values}},
             show_only=["templates/pod-template-file.yaml"],
             chart_dir=self.temp_chart_dir,
         )
-
-        assert {"name": "dags", "emptyDir": {}} in 
jmespath.search("spec.volumes", docs[0])
+        assert {"name": "dags", **expected} in jmespath.search("spec.volumes", 
docs[0])
 
     @pytest.mark.parametrize(
-        "log_persistence_values, expected",
+        "log_values, expected",
         [
-            ({"enabled": False}, {"emptyDir": {}}),
-            ({"enabled": True}, {"persistentVolumeClaim": {"claimName": 
"release-name-logs"}}),
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
+            (
+                {"persistence": {"enabled": False}, "emptyDirConfig": 
{"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+            (
+                {"persistence": {"enabled": True}},
+                {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+            ),
             (
-                {"enabled": True, "existingClaim": "test-claim"},
+                {"persistence": {"enabled": True, "existingClaim": 
"test-claim"}},
                 {"persistentVolumeClaim": {"claimName": "test-claim"}},
             ),
         ],
     )
-    def test_logs_persistence_changes_volume(self, log_persistence_values, 
expected):
+    def test_logs_persistence_changes_volume(self, log_values, expected):
         docs = render_chart(
-            values={"logs": {"persistence": log_persistence_values}},
+            values={"logs": log_values},
             show_only=["templates/pod-template-file.yaml"],
             chart_dir=self.temp_chart_dir,
         )
diff --git a/helm_tests/airflow_core/test_dag_processor.py 
b/helm_tests/airflow_core/test_dag_processor.py
index 2ab941f8e2..b9843c1594 100644
--- a/helm_tests/airflow_core/test_dag_processor.py
+++ b/helm_tests/airflow_core/test_dag_processor.py
@@ -368,20 +368,27 @@ class TestDagProcessor:
         )
 
     @pytest.mark.parametrize(
-        "log_persistence_values, expected_volume",
+        "log_values, expected_volume",
         [
-            ({"enabled": False}, {"emptyDir": {}}),
-            ({"enabled": True}, {"persistentVolumeClaim": {"claimName": 
"release-name-logs"}}),
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
             (
-                {"enabled": True, "existingClaim": "test-claim"},
+                {"persistence": {"enabled": False}, "emptyDirConfig": 
{"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+            (
+                {"persistence": {"enabled": True}},
+                {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+            ),
+            (
+                {"persistence": {"enabled": True, "existingClaim": 
"test-claim"}},
                 {"persistentVolumeClaim": {"claimName": "test-claim"}},
             ),
         ],
     )
-    def test_logs_persistence_changes_volume(self, log_persistence_values, 
expected_volume):
+    def test_logs_persistence_changes_volume(self, log_values, 
expected_volume):
         docs = render_chart(
             values={
-                "logs": {"persistence": log_persistence_values},
+                "logs": log_values,
                 "dagProcessor": {"enabled": True},
             },
             
show_only=["templates/dag-processor/dag-processor-deployment.yaml"],
diff --git a/helm_tests/airflow_core/test_scheduler.py 
b/helm_tests/airflow_core/test_scheduler.py
index d1007d8700..c7336bbc98 100644
--- a/helm_tests/airflow_core/test_scheduler.py
+++ b/helm_tests/airflow_core/test_scheduler.py
@@ -426,19 +426,26 @@ class TestScheduler:
         )
 
     @pytest.mark.parametrize(
-        "log_persistence_values, expected_volume",
+        "log_values, expected_volume",
         [
-            ({"enabled": False}, {"emptyDir": {}}),
-            ({"enabled": True}, {"persistentVolumeClaim": {"claimName": 
"release-name-logs"}}),
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
             (
-                {"enabled": True, "existingClaim": "test-claim"},
+                {"persistence": {"enabled": False}, "emptyDirConfig": 
{"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+            (
+                {"persistence": {"enabled": True}},
+                {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+            ),
+            (
+                {"persistence": {"enabled": True, "existingClaim": 
"test-claim"}},
                 {"persistentVolumeClaim": {"claimName": "test-claim"}},
             ),
         ],
     )
-    def test_logs_persistence_changes_volume(self, log_persistence_values, 
expected_volume):
+    def test_logs_persistence_changes_volume(self, log_values, 
expected_volume):
         docs = render_chart(
-            values={"logs": {"persistence": log_persistence_values}},
+            values={"logs": log_values},
             show_only=["templates/scheduler/scheduler-deployment.yaml"],
         )
 
diff --git a/helm_tests/airflow_core/test_triggerer.py 
b/helm_tests/airflow_core/test_triggerer.py
index 6375071849..3c227afdf5 100644
--- a/helm_tests/airflow_core/test_triggerer.py
+++ b/helm_tests/airflow_core/test_triggerer.py
@@ -389,21 +389,28 @@ class TestTriggerer:
         )
 
     @pytest.mark.parametrize(
-        "log_persistence_values, expected_volume",
+        "log_values, expected_volume",
         [
-            ({"enabled": False}, {"emptyDir": {}}),
-            ({"enabled": True}, {"persistentVolumeClaim": {"claimName": 
"release-name-logs"}}),
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
             (
-                {"enabled": True, "existingClaim": "test-claim"},
+                {"persistence": {"enabled": False}, "emptyDirConfig": 
{"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+            (
+                {"persistence": {"enabled": True}},
+                {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+            ),
+            (
+                {"persistence": {"enabled": True, "existingClaim": 
"test-claim"}},
                 {"persistentVolumeClaim": {"claimName": "test-claim"}},
             ),
         ],
     )
-    def test_logs_persistence_changes_volume(self, log_persistence_values, 
expected_volume):
+    def test_logs_persistence_changes_volume(self, log_values, 
expected_volume):
         docs = render_chart(
             values={
                 "triggerer": {"persistence": {"enabled": False}},
-                "logs": {"persistence": log_persistence_values},
+                "logs": log_values,
             },
             show_only=["templates/triggerer/triggerer-deployment.yaml"],
         )
diff --git a/helm_tests/airflow_core/test_worker.py 
b/helm_tests/airflow_core/test_worker.py
index 80682f9e17..6dfe99d7d4 100644
--- a/helm_tests/airflow_core/test_worker.py
+++ b/helm_tests/airflow_core/test_worker.py
@@ -456,22 +456,29 @@ class TestWorker:
         assert livenessprobe is None
 
     @pytest.mark.parametrize(
-        "log_persistence_values, expected_volume",
+        "log_values, expected_volume",
         [
-            ({"enabled": False}, {"emptyDir": {}}),
-            ({"enabled": True}, {"persistentVolumeClaim": {"claimName": 
"release-name-logs"}}),
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
             (
-                {"enabled": True, "existingClaim": "test-claim"},
+                {"persistence": {"enabled": False}, "emptyDirConfig": 
{"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+            (
+                {"persistence": {"enabled": True}},
+                {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+            ),
+            (
+                {"persistence": {"enabled": True, "existingClaim": 
"test-claim"}},
                 {"persistentVolumeClaim": {"claimName": "test-claim"}},
             ),
         ],
     )
-    def test_logs_persistence_changes_volume(self, log_persistence_values, 
expected_volume):
+    def test_logs_persistence_changes_volume(self, log_values, 
expected_volume):
         docs = render_chart(
             values={
                 "executor": "CeleryExecutor",
                 "workers": {"persistence": {"enabled": False}},
-                "logs": {"persistence": log_persistence_values},
+                "logs": log_values,
             },
             show_only=["templates/workers/worker-deployment.yaml"],
         )
diff --git a/helm_tests/other/test_redis.py b/helm_tests/other/test_redis.py
index 466e7b6bf1..82266e3f47 100644
--- a/helm_tests/other/test_redis.py
+++ b/helm_tests/other/test_redis.py
@@ -340,6 +340,23 @@ class TestRedis:
         )
         assert {"foo": "bar"} == 
jmespath.search("spec.volumeClaimTemplates[0].metadata.annotations", docs[0])
 
+    @pytest.mark.parametrize(
+        "redis_values, expected",
+        [
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
+            (
+                {"persistence": {"enabled": False}, "emptyDirConfig": 
{"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+        ],
+    )
+    def test_should_use_empty_dir_on_persistence_disabled(self, redis_values, 
expected):
+        docs = render_chart(
+            values={"redis": redis_values},
+            show_only=["templates/redis/redis-statefulset.yaml"],
+        )
+        assert {"name": "redis-db", **expected} in 
jmespath.search("spec.template.spec.volumes", docs[0])
+
 
 class TestRedisServiceAccount:
     """Tests redis service account."""

Reply via email to