This is an automated email from the ASF dual-hosted git repository.
Miretpl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 715043b8980 Honor config_file in deferrable KubernetesJobOperator
(#76712)
715043b8980 is described below
commit 715043b898089c40053108246a0e549f9c16c982
Author: rjgoyln <[email protected]>
AuthorDate: Tue Nov 17 06:35:04 2026 +0800
Honor config_file in deferrable KubernetesJobOperator (#76712)
* Honor config_file in deferrable Kubernetes tasks
AsyncKubernetesHook accepted config_file but never read it, so a
deferrable task configured with an explicit kubeconfig authenticated
against whatever the default location resolved to. Inside a single
KubernetesJobTrigger the two hooks disagreed: the sync pod manager used
the requested file while the async hook did not.
Resolving config_file the way the sync hook does also brings it into the
mutual-exclusion check, so combining it with in_cluster, kube_config or
config_dict now fails instead of silently picking one.
* Test the config_file reload behavior rather than the cache flag
Asserting on the hook's private cache flag only showed that a flag was
left unset; it never showed that the kubeconfig is re-read on the next
call, which is what a triggerer depends on when an exec plugin's token
expires part-way through a job. Awaiting the load twice pins that
directly and survives any change to how the caching is implemented.
The other test's name described how many arguments it passes instead of
what it checks.
* Pin only the config_file lookup in the async hook test
The second load and the exec-auth kubeconfig checked that _load_config
re-reads the kubeconfig on every call. Main already does that
unconditionally, so reverting the fix could never break that half of
the assertion; it only made the test look like it covered token refresh.
* Scope the config_file changelog note to the affected operator
Deferrable KubernetesPodOperator already honored config_file by turning
it into config_dict before deferring, so "deferrable tasks" overstated
the change. The old fallback was also the connection's kube_config_path
when set, not only the default kubeconfig.
Before this fix the job trigger read config_file only with
do_xcom_push; now it always does, so deployments that ship the file to
workers but not to the triggerer need to know.
* Name config_file in the Kubernetes hooks' mutual-exclusion error
config_file can now trip the async hook's check, and it already could in
the sync one, but the error listed only kube_config_path. Someone who
passed config_file with in_cluster got back a list of options they never
set.
* Tell users how to give the triggerer the kubeconfig
The note said the triggerer must be able to read config_file but not
what to do when it cannot. The connection's kube_config_path is no way
out, since the triggerer resolves that path as well; the kube_config
extra is, because it carries the contents rather than a path.
---------
Co-authored-by: Przemysław Mirowski
<[email protected]>
---
providers/cncf/kubernetes/docs/changelog.rst | 11 ++++++
.../providers/cncf/kubernetes/hooks/kubernetes.py | 8 ++--
.../unit/cncf/kubernetes/hooks/test_kubernetes.py | 43 +++++++++++++++++++---
3 files changed, 53 insertions(+), 9 deletions(-)
diff --git a/providers/cncf/kubernetes/docs/changelog.rst
b/providers/cncf/kubernetes/docs/changelog.rst
index a3c0c19545a..7ec63ca253a 100644
--- a/providers/cncf/kubernetes/docs/changelog.rst
+++ b/providers/cncf/kubernetes/docs/changelog.rst
@@ -27,6 +27,17 @@
Changelog
---------
+.. note::
+ ``AsyncKubernetesHook`` now honors its ``config_file`` argument, so a
deferrable
+ ``KubernetesJobOperator`` authenticates against the kubeconfig it was
given rather than the
+ connection's ``kube_config_path`` or the default kubeconfig. The triggerer
must therefore be
+ able to read that file: either make it available on the triggerer at the
same path, or drop
+ ``config_file`` and put the kubeconfig contents in the connection's
``kube_config`` extra.
+ The connection's ``kube_config_path`` does not help here, since the
triggerer resolves that
+ path too. Because ``config_file`` now takes part in the mutual-exclusion
check, combining it
+ with ``in_cluster``, ``kube_config`` or ``config_dict`` raises instead of
being ignored — drop
+ whichever option you did not mean to use.
+
11.0.0
......
diff --git
a/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py
b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py
index 07f82372cf0..085ec397ffb 100644
---
a/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py
+++
b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py
@@ -263,8 +263,8 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
if num_selected_configuration > 1:
raise AirflowException(
- "Invalid connection configuration. Options kube_config_path, "
- "kube_config, in_cluster, config_dict are mutually exclusive. "
+ "Invalid connection configuration. Options
config_file/kube_config_path, "
+ "kube_config, in_cluster, and config_dict are mutually
exclusive. "
"You can only use one option at a time."
)
@@ -935,7 +935,7 @@ class AsyncKubernetesHook(KubernetesHook):
in_cluster = self._coalesce_param(self.in_cluster, await
self._get_field("in_cluster"))
cluster_context = self._coalesce_param(self.cluster_context, await
self._get_field("cluster_context"))
- kubeconfig_path = await self._get_field("kube_config_path")
+ kubeconfig_path = self._coalesce_param(self.config_file, await
self._get_field("kube_config_path"))
kubeconfig = await self._get_field("kube_config")
num_selected_configuration = sum(
@@ -944,7 +944,7 @@ class AsyncKubernetesHook(KubernetesHook):
if num_selected_configuration > 1:
raise AirflowException(
- "Invalid connection configuration. Options kube_config_path, "
+ "Invalid connection configuration. Options
config_file/kube_config_path, "
"kube_config, in_cluster, and config_dict are mutually
exclusive. "
"You can only use one option at a time."
)
diff --git
a/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/hooks/test_kubernetes.py
b/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/hooks/test_kubernetes.py
index df74354f83b..bfb9b769fd5 100644
---
a/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/hooks/test_kubernetes.py
+++
b/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/hooks/test_kubernetes.py
@@ -63,6 +63,9 @@ HOOK_MODULE =
"airflow.providers.cncf.kubernetes.hooks.kubernetes"
CONN_ID = "kubernetes-test-id"
ASYNC_CONFIG_PATH = "/files/path/to/config/file"
+MUTUALLY_EXCLUSIVE_CONFIG_ERROR = (
+ "config_file/kube_config_path, kube_config, in_cluster, and config_dict
are mutually exclusive"
+)
POD_NAME = "test-pod"
NAMESPACE = "test-namespace"
JOB_NAME = "test-job"
@@ -1049,7 +1052,7 @@ class TestKubernetesHookIncorrectConfiguration:
kubernetes_hook = KubernetesHook()
with (
mock.patch.dict("os.environ",
AIRFLOW_CONN_KUBERNETES_DEFAULT=conn_uri),
- pytest.raises(AirflowException, match="Invalid connection
configuration"),
+ pytest.raises(AirflowException,
match=MUTUALLY_EXCLUSIVE_CONFIG_ERROR),
):
kubernetes_hook.get_conn()
@@ -1262,14 +1265,44 @@ class TestAsyncKubernetesHook:
assert kube_client is None
@pytest.mark.asyncio
- async def test_load_config_with_several_params(self,
sdk_connection_not_found):
+ @pytest.mark.parametrize(
+ "connection_extras",
+ [
+ pytest.param({}, id="no_connection_extras"),
+ pytest.param({"kube_config_path": ASYNC_CONFIG_PATH},
id="overrides_connection_kube_config_path"),
+ ],
+ )
+ @mock.patch(f"{HOOK_MODULE}.async_config.load_kube_config", autospec=True)
+ async def test_load_config_with_config_file(self, mock_load_kube_config,
connection_extras, tmp_path):
+ kubeconfig_file = tmp_path / "config"
+ kubeconfig_file.write_text("{}")
+
+ hook = AsyncKubernetesHook(connection_extras=connection_extras,
config_file=str(kubeconfig_file))
+ await hook._load_config()
+
+ mock_load_kube_config.assert_awaited_once_with(
+ config_file=str(kubeconfig_file),
+ client_configuration=hook.client_configuration,
+ context=None,
+ )
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ ("competing_kwargs", "connection_extras"),
+ [
+ pytest.param({"in_cluster": True}, {}, id="in_cluster"),
+ pytest.param({"config_dict": {"test": "kube"}}, {},
id="config_dict"),
+ pytest.param({}, {"kube_config": '{"test": "kube"}'},
id="kube_config"),
+ ],
+ )
+ async def test_load_config_with_mutually_exclusive_params(self,
competing_kwargs, connection_extras):
hook = AsyncKubernetesHook(
- conn_id=CONN_ID,
- in_cluster=True,
+ connection_extras=connection_extras,
config_file=ASYNC_CONFIG_PATH,
cluster_context=None,
+ **competing_kwargs,
)
- with pytest.raises(AirflowException):
+ with pytest.raises(AirflowException,
match=MUTUALLY_EXCLUSIVE_CONFIG_ERROR):
await hook._load_config()
@pytest.mark.parametrize(