This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new 06e53f2 Chart: configurable number of retention days for log groomers (#17764) 06e53f2 is described below commit 06e53f26e5cb2f1ad4aabe05fa12d2db9c66e282 Author: cccs-seb <64980897+cccs-...@users.noreply.github.com> AuthorDate: Tue Aug 24 19:34:06 2021 -0400 Chart: configurable number of retention days for log groomers (#17764) --- .../templates/scheduler/scheduler-deployment.yaml | 5 ++++ chart/templates/workers/worker-deployment.yaml | 5 ++++ chart/tests/test_scheduler.py | 30 ++++++++++++++++++++++ chart/tests/test_worker.py | 30 ++++++++++++++++++++++ chart/values.schema.json | 10 ++++++++ chart/values.yaml | 4 +++ 6 files changed, 84 insertions(+) diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 8c51fd7..25d2554 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -211,6 +211,11 @@ spec: {{- if .Values.scheduler.logGroomerSidecar.args }} args: {{ tpl (toYaml .Values.scheduler.logGroomerSidecar.args) . | nindent 12 }} {{- end }} + {{ if .Values.scheduler.logGroomerSidecar.retentionDays }} + env: + - name: AIRFLOW__LOG_RETENTION_DAYS + value: "{{ .Values.scheduler.logGroomerSidecar.retentionDays }}" + {{- end }} volumeMounts: - name: logs mountPath: {{ template "airflow_logs" . }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 787d387..64abb1a 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -200,6 +200,11 @@ spec: {{- if .Values.workers.logGroomerSidecar.args }} args: {{ tpl (toYaml .Values.workers.logGroomerSidecar.args) . | nindent 12 }} {{- end }} + {{ if .Values.workers.logGroomerSidecar.retentionDays }} + env: + - name: AIRFLOW__LOG_RETENTION_DAYS + value: "{{ .Values.workers.logGroomerSidecar.retentionDays }}" + {{- end }} resources: {{ toYaml .Values.workers.logGroomerSidecar.resources | indent 12 }} volumeMounts: diff --git a/chart/tests/test_scheduler.py b/chart/tests/test_scheduler.py index 0c06163..1225790 100644 --- a/chart/tests/test_scheduler.py +++ b/chart/tests/test_scheduler.py @@ -354,6 +354,14 @@ class SchedulerTest(unittest.TestCase): assert jmespath.search("spec.template.spec.containers[1].command", docs[0]) is None assert ["bash", "/clean-logs"] == jmespath.search("spec.template.spec.containers[1].args", docs[0]) + def test_log_groomer_collector_default_retention_days(self): + docs = render_chart(show_only=["templates/scheduler/scheduler-deployment.yaml"]) + + assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search( + "spec.template.spec.containers[1].env[0].name", docs[0] + ) + assert "15" == jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0]) + @parameterized.expand( [ (None, None), @@ -389,6 +397,28 @@ class SchedulerTest(unittest.TestCase): @parameterized.expand( [ + (None, None), + (30, "30"), + ] + ) + def test_log_groomer_retention_days_overrides(self, retention_days, retention_result): + docs = render_chart( + values={"scheduler": {"logGroomerSidecar": {"retentionDays": retention_days}}}, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + ) + + if retention_result: + assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search( + "spec.template.spec.containers[1].env[0].name", docs[0] + ) + assert retention_result == jmespath.search( + "spec.template.spec.containers[1].env[0].value", docs[0] + ) + else: + assert jmespath.search("spec.template.spec.containers[1].env", docs[0]) is None + + @parameterized.expand( + [ ({"gitSync": {"enabled": True}},), ({"gitSync": {"enabled": True}, "persistence": {"enabled": True}},), ] diff --git a/chart/tests/test_worker.py b/chart/tests/test_worker.py index 8f4685d..7e79389 100644 --- a/chart/tests/test_worker.py +++ b/chart/tests/test_worker.py @@ -423,6 +423,14 @@ class WorkerTest(unittest.TestCase): assert jmespath.search("spec.template.spec.containers[1].command", docs[0]) is None assert ["bash", "/clean-logs"] == jmespath.search("spec.template.spec.containers[1].args", docs[0]) + def test_log_groomer_collector_default_retention_days(self): + docs = render_chart(show_only=["templates/workers/worker-deployment.yaml"]) + + assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search( + "spec.template.spec.containers[1].env[0].name", docs[0] + ) + assert "15" == jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0]) + @parameterized.expand( [ (None, None), @@ -456,6 +464,28 @@ class WorkerTest(unittest.TestCase): assert ["RELEASE-NAME"] == jmespath.search("spec.template.spec.containers[1].command", docs[0]) assert ["Helm"] == jmespath.search("spec.template.spec.containers[1].args", docs[0]) + @parameterized.expand( + [ + (None, None), + (30, "30"), + ] + ) + def test_log_groomer_retention_days_overrides(self, retention_days, retention_result): + docs = render_chart( + values={"workers": {"logGroomerSidecar": {"retentionDays": retention_days}}}, + show_only=["templates/workers/worker-deployment.yaml"], + ) + + if retention_result: + assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search( + "spec.template.spec.containers[1].env[0].name", docs[0] + ) + assert retention_result == jmespath.search( + "spec.template.spec.containers[1].env[0].value", docs[0] + ) + else: + assert jmespath.search("spec.template.spec.containers[1].env", docs[0]) is None + def test_dags_gitsync_sidecar_and_init_container(self): docs = render_chart( values={"dags": {"gitSync": {"enabled": True}}}, diff --git a/chart/values.schema.json b/chart/values.schema.json index 3c21bf2..491404b 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1132,6 +1132,11 @@ "/clean-logs" ] }, + "retentionDays": { + "description": "Number of days to retain the logs when running the Airflow workers log groomer sidecar.", + "type": "integer", + "default": 15 + }, "resources": { "description": "Resources for Airflow workers log groomer sidecar.", "type": "object", @@ -1375,6 +1380,11 @@ "/clean-logs" ] }, + "retentionDays": { + "description": "Number of days to retain the logs when running the Airflow scheduler log groomer sidecar.", + "type": "integer", + "default": 15 + }, "resources": { "description": "Resources for log groomer sidecar.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index a634d1b..a7cbd19d 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -453,6 +453,8 @@ workers: command: ~ # Args to use when running the Airflow worker log groomer sidecar (templated). args: ["bash", "/clean-logs"] + # Number of days to retain logs + retentionDays: 15 resources: {} # limits: # cpu: 100m @@ -546,6 +548,8 @@ scheduler: command: ~ # Args to use when running the Airflow scheduler log groomer sidecar (templated). args: ["bash", "/clean-logs"] + # Number of days to retain logs + retentionDays: 15 resources: {} # limits: # cpu: 100m