This is an automated email from the ASF dual-hosted git repository.
Miretpl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new dd39c346e69 Stop default Celery worker Service selecting
KubernetesExecutor pods (#74587)
dd39c346e69 is described below
commit dd39c346e693e5711467b98c4e77207e4f16493a
Author: Baris Can Vural <[email protected]>
AuthorDate: Sun Nov 15 13:46:21 2026 +0300
Stop default Celery worker Service selecting KubernetesExecutor pods
(#74587)
* Stop default Celery worker Service selecting KubernetesExecutor pods
When the executor lists both KubernetesExecutor and CeleryExecutor, task
pods
created from the pod template carry the same tier/component/release labels
as
the default worker set's Celery pods, so the headless worker Service picked
them
up as endpoints. Its endpoints then changed on every task start and finish:
anything watching them (for example a service mesh control plane pushing
config
to every sidecar on each change) paid for that churn, and the Service's DNS
name
resolved to pods that do not serve worker logs.
Named worker sets never had this problem because of their worker-set label.
The
workload selector stays unchanged since it is immutable, so existing
releases
upgrade in place.
* Use jmespath in worker Service selector test
Other chart tests read rendered manifests through jmespath, so the new test
should follow the same convention, as requested in review.
---
chart/templates/workers/worker-deployment.yaml | 4 ++--
chart/templates/workers/worker-service.yaml | 2 --
chart/tests/helm_tests/airflow_core/test_worker.py | 20 ++++++++++++++++++++
3 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/chart/templates/workers/worker-deployment.yaml
b/chart/templates/workers/worker-deployment.yaml
index aae67b42034..78a6fac473f 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -104,9 +104,9 @@ spec:
tier: airflow
component: worker
release: {{ .Release.Name }}
- {{- if ne .Values.workers.celery.name "default" }}
+ {{- /* Set for every worker set, unlike the immutable spec.selector,
so the worker Service can tell
+ Celery workers apart from KubernetesExecutor pods, which carry the
same labels above. */}}
worker-set: {{ .Values.workers.celery.name }}
- {{- end }}
{{- if or .Values.labels .Values.workers.celery.labels }}
{{- mustMerge .Values.workers.celery.labels .Values.labels | toYaml
| nindent 8 }}
{{- end }}
diff --git a/chart/templates/workers/worker-service.yaml
b/chart/templates/workers/worker-service.yaml
index 940284f3a94..7d2e14a1e7e 100644
--- a/chart/templates/workers/worker-service.yaml
+++ b/chart/templates/workers/worker-service.yaml
@@ -50,9 +50,7 @@ spec:
tier: airflow
component: worker
release: {{ .Release.Name }}
- {{- if ne .Values.workers.celery.name "default" }}
worker-set: {{ .Values.workers.celery.name }}
- {{- end }}
ports:
- name: worker-logs
protocol: TCP
diff --git a/chart/tests/helm_tests/airflow_core/test_worker.py
b/chart/tests/helm_tests/airflow_core/test_worker.py
index c069bc4cdf2..e09e51dd19e 100644
--- a/chart/tests/helm_tests/airflow_core/test_worker.py
+++ b/chart/tests/helm_tests/airflow_core/test_worker.py
@@ -18,6 +18,7 @@ from __future__ import annotations
import jmespath
import pytest
+import yaml
from chart_utils.helm_template_generator import HelmFailedError, render_chart
from chart_utils.log_groomer import LogGroomerTestBase
@@ -1843,6 +1844,25 @@ class TestWorkerService:
assert jmespath.search("spec.ipFamilyPolicy", docs[0]) is None
assert jmespath.search("spec.ipFamilies", docs[0]) is None
+ def test_selector_excludes_kubernetes_executor_pods(self):
+ docs = render_chart(
+ values={"executor": "KubernetesExecutor,CeleryExecutor"},
+ show_only=[
+ "templates/configmaps/configmap.yaml",
+ "templates/workers/worker-deployment.yaml",
+ "templates/workers/worker-service.yaml",
+ ],
+ )
+
+ configmap, worker, service = docs
+ selector = jmespath.search("spec.selector", service).items()
+ celery_pod_labels = jmespath.search("spec.template.metadata.labels",
worker).items()
+ pod_template =
yaml.safe_load(jmespath.search('data."pod_template_file.yaml"', configmap))
+ kubernetes_pod_labels = jmespath.search("metadata.labels",
pod_template).items()
+
+ assert selector <= celery_pod_labels
+ assert not selector <= kubernetes_pod_labels
+
class TestWorkerCeleryServiceAccount:
def test_should_not_create_service_account_when_disabled(self):