This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new 279e8b7772 Chart: Allow valueFrom in env config of components (#40135) 279e8b7772 is described below commit 279e8b7772c483890726734600df89613cca3263 Author: Daniel Gellert <dab.d...@gmail.com> AuthorDate: Tue Jun 11 02:21:07 2024 +0200 Chart: Allow valueFrom in env config of components (#40135) --- chart/templates/_helpers.yaml | 26 ++ chart/values.schema.json | 320 +++++++++++++++++++++-- helm_tests/airflow_aux/test_create_user_job.py | 20 +- helm_tests/airflow_aux/test_pod_template_file.py | 24 +- helm_tests/airflow_core/test_dag_processor.py | 20 +- helm_tests/airflow_core/test_scheduler.py | 20 +- helm_tests/airflow_core/test_triggerer.py | 20 +- helm_tests/airflow_core/test_worker.py | 20 +- helm_tests/other/test_flower.py | 20 +- helm_tests/webserver/test_webserver.py | 20 +- 10 files changed, 486 insertions(+), 24 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index aaa86ecbe8..dfa23fba2f 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -1025,10 +1025,36 @@ capabilities: {{- $env := index . 1 -}} {{- range $i, $config := $env }} - name: {{ $config.name }} + {{- if $config.value }} value: {{ $config.value | quote }} + {{- else if $config.valueFrom }} + valueFrom: + {{- if $config.valueFrom.secretKeyRef }} + secretKeyRef: + name: {{ $config.valueFrom.secretKeyRef.name }} + key: {{ $config.valueFrom.secretKeyRef.key }} + {{- else if $config.valueFrom.configMapKeyRef }} + configMapKeyRef: + name: {{ $config.valueFrom.configMapKeyRef.name }} + key: {{ $config.valueFrom.configMapKeyRef.key }} + {{- end }} + {{- end }} {{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }} - name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }} + {{- if $config.value }} value: {{ $config.value | quote }} + {{- else if $config.valueFrom }} + valueFrom: + {{- if $config.valueFrom.secretKeyRef }} + secretKeyRef: + name: {{ $config.valueFrom.secretKeyRef.name }} + key: {{ $config.valueFrom.secretKeyRef.key }} + {{- else if $config.valueFrom.configMapKeyRef }} + configMapKeyRef: + name: {{ $config.valueFrom.configMapKeyRef.name }} + key: {{ $config.valueFrom.configMapKeyRef.key }} + {{- end }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index e155719720..c2abb29441 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -2145,11 +2145,47 @@ }, "value": { "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] } }, "required": [ - "name", - "value" + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } ], "additionalProperties": false } @@ -2693,11 +2729,47 @@ }, "value": { "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] } }, "required": [ - "name", - "value" + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } ], "additionalProperties": false } @@ -3164,11 +3236,47 @@ }, "value": { "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] } }, "required": [ - "name", - "value" + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } ], "additionalProperties": false } @@ -3647,11 +3755,47 @@ }, "value": { "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] } }, "required": [ - "name", - "value" + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } ], "additionalProperties": false } @@ -3952,11 +4096,47 @@ }, "value": { "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] } }, "required": [ - "name", - "value" + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } ], "additionalProperties": false } @@ -5681,11 +5861,47 @@ }, "value": { "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] } }, "required": [ - "name", - "value" + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } ], "additionalProperties": false } @@ -6174,11 +6390,47 @@ }, "value": { "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] } }, "required": [ - "name", - "value" + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } ], "additionalProperties": false } @@ -7798,11 +8050,47 @@ }, "value": { "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] } }, "required": [ - "name", - "value" + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } ], "additionalProperties": false } diff --git a/helm_tests/airflow_aux/test_create_user_job.py b/helm_tests/airflow_aux/test_create_user_job.py index 6ee458bcbb..fb90d7c599 100644 --- a/helm_tests/airflow_aux/test_create_user_job.py +++ b/helm_tests/airflow_aux/test_create_user_job.py @@ -228,7 +228,17 @@ class TestCreateUserJob: docs = render_chart( values={ "createUserJob": { - "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ], }, }, show_only=["templates/jobs/create-user-job.yaml"], @@ -237,6 +247,14 @@ class TestCreateUserJob: assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( "spec.template.spec.containers[0].env", docs[0] ) + assert { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) + assert { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) def test_should_enable_custom_env(self): docs = render_chart( diff --git a/helm_tests/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py index 8216347512..1efccbf410 100644 --- a/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -767,7 +767,21 @@ class TestPodTemplateFile: def test_should_add_extraEnvs(self): docs = render_chart( - values={"workers": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}}, + values={ + "workers": { + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ] + } + }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -775,6 +789,14 @@ class TestPodTemplateFile: assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( "spec.containers[0].env", docs[0] ) + assert { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + } in jmespath.search("spec.containers[0].env", docs[0]) + assert { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + } in jmespath.search("spec.containers[0].env", docs[0]) def test_should_add_component_specific_labels(self): docs = render_chart( diff --git a/helm_tests/airflow_core/test_dag_processor.py b/helm_tests/airflow_core/test_dag_processor.py index 31256b4095..9ecf06d36c 100644 --- a/helm_tests/airflow_core/test_dag_processor.py +++ b/helm_tests/airflow_core/test_dag_processor.py @@ -203,7 +203,17 @@ class TestDagProcessor: values={ "dagProcessor": { "enabled": True, - "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ], }, }, show_only=["templates/dag-processor/dag-processor-deployment.yaml"], @@ -212,6 +222,14 @@ class TestDagProcessor: assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( "spec.template.spec.containers[0].env", docs[0] ) + assert { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) + assert { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) def test_should_add_extraEnvs_to_wait_for_migration_container(self): docs = render_chart( diff --git a/helm_tests/airflow_core/test_scheduler.py b/helm_tests/airflow_core/test_scheduler.py index eef2f87564..3fb7f6754d 100644 --- a/helm_tests/airflow_core/test_scheduler.py +++ b/helm_tests/airflow_core/test_scheduler.py @@ -169,7 +169,17 @@ class TestScheduler: docs = render_chart( values={ "scheduler": { - "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ], }, }, show_only=["templates/scheduler/scheduler-deployment.yaml"], @@ -178,6 +188,14 @@ class TestScheduler: assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( "spec.template.spec.containers[0].env", docs[0] ) + assert { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) + assert { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) def test_should_add_extraEnvs_to_wait_for_migration_container(self): docs = render_chart( diff --git a/helm_tests/airflow_core/test_triggerer.py b/helm_tests/airflow_core/test_triggerer.py index a684e0f334..0d8c4dd820 100644 --- a/helm_tests/airflow_core/test_triggerer.py +++ b/helm_tests/airflow_core/test_triggerer.py @@ -191,7 +191,17 @@ class TestTriggerer: docs = render_chart( values={ "triggerer": { - "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ], } }, show_only=["templates/triggerer/triggerer-deployment.yaml"], @@ -200,6 +210,14 @@ class TestTriggerer: assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( "spec.template.spec.containers[0].env", docs[0] ) + assert { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) + assert { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) def test_should_add_extraEnvs_to_wait_for_migration_container(self): docs = render_chart( diff --git a/helm_tests/airflow_core/test_worker.py b/helm_tests/airflow_core/test_worker.py index bbfea0be68..af370f3d9a 100644 --- a/helm_tests/airflow_core/test_worker.py +++ b/helm_tests/airflow_core/test_worker.py @@ -182,7 +182,17 @@ class TestWorker: docs = render_chart( values={ "workers": { - "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ], }, }, show_only=["templates/workers/worker-deployment.yaml"], @@ -191,6 +201,14 @@ class TestWorker: assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( "spec.template.spec.containers[0].env", docs[0] ) + assert { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) + assert { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) def test_should_add_extraEnvs_to_wait_for_migration_container(self): docs = render_chart( diff --git a/helm_tests/other/test_flower.py b/helm_tests/other/test_flower.py index b88fd25ef3..0f2f2dd66b 100644 --- a/helm_tests/other/test_flower.py +++ b/helm_tests/other/test_flower.py @@ -305,7 +305,17 @@ class TestFlowerDeployment: values={ "flower": { "enabled": True, - "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ], } }, show_only=["templates/flower/flower-deployment.yaml"], @@ -314,6 +324,14 @@ class TestFlowerDeployment: assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( "spec.template.spec.containers[0].env", docs[0] ) + assert { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) + assert { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) def test_should_add_component_specific_labels(self): docs = render_chart( diff --git a/helm_tests/webserver/test_webserver.py b/helm_tests/webserver/test_webserver.py index 364a7e4b16..3d34804211 100644 --- a/helm_tests/webserver/test_webserver.py +++ b/helm_tests/webserver/test_webserver.py @@ -219,7 +219,17 @@ class TestWebserverDeployment: docs = render_chart( values={ "webserver": { - "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}], + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ], }, }, show_only=["templates/webserver/webserver-deployment.yaml"], @@ -228,6 +238,14 @@ class TestWebserverDeployment: assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( "spec.template.spec.containers[0].env", docs[0] ) + assert { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) + assert { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) def test_should_add_extra_volume_and_extra_volume_mount(self): docs = render_chart(