This is an automated email from the ASF dual-hosted git repository.
ashb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f1b36db6093 Back off between GitDagBundle bare clone attempts so a
fresh GitHub App token can propagate (#73878)
f1b36db6093 is described below
commit f1b36db6093ceae6c46a5ae053f4adc8745208b5
Author: MichalJaroslawKrzywanski-TomTom <[email protected]>
AuthorDate: Thu Oct 1 14:05:42 2026 +0200
Back off between GitDagBundle bare clone attempts so a fresh GitHub App
token can propagate (#73878)
* Back off between GitDagBundle bare clone attempts so a fresh GitHub App
token can propagate
GitHub rejects a just-issued App installation token with "remote:
Repository not found." for a few seconds; the same clone succeeds
unchanged about 10 s later. Every KubernetesExecutor task pod mints its
own token in GitHook and immediately calls _clone_bare_repo_if_required,
whose retry allowed two attempts with no wait, so both landed inside that
window and the task died at startup with "Error cloning repository".
Give the bare clone five attempts with exponential backoff (2, 4, 8, 15 s)
so at least one lands after the token is usable. Tests stub the retry's
sleep so the persistent-failure cases do not spend that time, and a new
test drives the real bundle code against a clone that fails twice with
"Repository not found" and asserts it waits and recovers.
closes: #73877
Co-Authored-By: Claude <[email protected]>
* Back off between bare clone attempts only for GitHub App connections
The retry wait only helps the auth path that mints a token right before
the clone; an SSH or PAT connection whose repository can never be cloned
would otherwise spend ~29 s failing instead of ~1 s. Expose the check as
GitHook.uses_github_app_auth and pick the tenacity stop and wait per
bundle from it: five attempts with a 2, 4, 8, 15 s wait for GitHub App
connections, the existing two immediate attempts for everything else.
Replace the test that asserted tenacity's own backoff behaviour with one
that asserts our branching: which auth path gets the extra attempts and
which one does not wait at all.
Co-Authored-By: krzywans <[email protected]>
Co-Authored-By: Claude <[email protected]>
---------
Co-authored-by: Claude <[email protected]>
---
.../git/src/airflow/providers/git/bundles/git.py | 31 ++++++++++++++-
.../git/src/airflow/providers/git/hooks/git.py | 7 +++-
providers/git/tests/unit/git/bundles/test_git.py | 44 ++++++++++++++++++++++
providers/git/tests/unit/git/hooks/test_git.py | 10 +++++
4 files changed, 89 insertions(+), 3 deletions(-)
diff --git a/providers/git/src/airflow/providers/git/bundles/git.py
b/providers/git/src/airflow/providers/git/bundles/git.py
index 69aeecf4daf..8a4644ba3c6 100644
--- a/providers/git/src/airflow/providers/git/bundles/git.py
+++ b/providers/git/src/airflow/providers/git/bundles/git.py
@@ -20,12 +20,13 @@ import os
import shutil
from contextlib import nullcontext
from pathlib import Path
+from typing import TYPE_CHECKING
from urllib.parse import urlparse
import structlog
from git import Repo
from git.exc import BadName, GitCommandError, InvalidGitRepositoryError,
NoSuchPathError
-from tenacity import retry, retry_if_exception_type, stop_after_attempt
+from tenacity import retry, retry_if_exception_type, stop_after_attempt,
wait_exponential
from airflow.dag_processing.bundles.base import BaseDagBundle
from airflow.providers.common.compat.sdk import AirflowException
@@ -35,8 +36,30 @@ from airflow.providers.git.hooks.git import GitHook
if AIRFLOW_V_3_3_PLUS:
from airflow.dag_processing.bundles.base import BundleVersion
+if TYPE_CHECKING:
+ from tenacity import RetryCallState
+
log = structlog.get_logger(__name__)
+# GitHub rejects a just-issued App installation token with "Repository not
found" for a few seconds,
+# and a fresh GitHook mints one right before the bare clone. Only that auth
path gets extra attempts
+# with a wait in between (2, 4, 8, 15 s); every other one keeps two immediate
attempts, so a repository
+# that can never be cloned still fails fast.
+_CLONE_STOP = stop_after_attempt(2)
+_GITHUB_APP_CLONE_STOP = stop_after_attempt(5)
+_GITHUB_APP_CLONE_WAIT = wait_exponential(multiplier=2, max=15)
+
+
+def _bare_clone_stop(retry_state: RetryCallState) -> bool:
+ bundle: GitDagBundle = retry_state.args[0]
+ stop = _GITHUB_APP_CLONE_STOP if bundle._uses_github_app_auth() else
_CLONE_STOP
+ return stop(retry_state)
+
+
+def _bare_clone_wait(retry_state: RetryCallState) -> float:
+ bundle: GitDagBundle = retry_state.args[0]
+ return _GITHUB_APP_CLONE_WAIT(retry_state) if
bundle._uses_github_app_auth() else 0
+
class GitDagBundle(BaseDagBundle):
"""
@@ -287,9 +310,13 @@ class GitDagBundle(BaseDagBundle):
shutil.rmtree(self.repo_path)
raise
+ def _uses_github_app_auth(self) -> bool:
+ return self.hook is not None and self.hook.uses_github_app_auth
+
@retry(
retry=retry_if_exception_type((InvalidGitRepositoryError,
GitCommandError)),
- stop=stop_after_attempt(2),
+ stop=_bare_clone_stop,
+ wait=_bare_clone_wait,
reraise=True,
)
def _clone_bare_repo_if_required(self) -> None:
diff --git a/providers/git/src/airflow/providers/git/hooks/git.py
b/providers/git/src/airflow/providers/git/hooks/git.py
index c802cb169ad..010e889e4d1 100644
--- a/providers/git/src/airflow/providers/git/hooks/git.py
+++ b/providers/git/src/airflow/providers/git/hooks/git.py
@@ -235,6 +235,11 @@ class GitHook(BaseHook):
return " ".join(parts)
+ @property
+ def uses_github_app_auth(self) -> bool:
+ """Whether the connection authenticates as a GitHub App and mints its
own installation tokens."""
+ return self.github_app_id is not None and self.github_installation_id
is not None
+
def _get_github_app_token(self):
try:
from github import Auth, GithubIntegration
@@ -392,7 +397,7 @@ printf 'username=%s\npassword=%s\n' "$AIRFLOW_GIT_USER"
"$AIRFLOW_GIT_TOKEN"
@contextlib.contextmanager
def configure_hook_env(self):
- if self.github_app_id is not None and self.github_installation_id is
not None:
+ if self.uses_github_app_auth:
self._ensure_github_app_token()
with self._token_credential_env():
yield
diff --git a/providers/git/tests/unit/git/bundles/test_git.py
b/providers/git/tests/unit/git/bundles/test_git.py
index 4eac57c1920..9849614a232 100644
--- a/providers/git/tests/unit/git/bundles/test_git.py
+++ b/providers/git/tests/unit/git/bundles/test_git.py
@@ -47,6 +47,13 @@ def _version_str(version_result):
return version_result
[email protected](autouse=True)
+def no_retry_wait():
+ """The bare clone backs off between retries; do not spend that wall-clock
time in tests."""
+ with mock.patch.object(GitDagBundle._clone_bare_repo_if_required.retry,
"sleep"):
+ yield
+
+
@pytest.fixture(autouse=True)
def bundle_temp_dir(tmp_path):
with conf_vars({("dag_processor", "dag_bundle_storage_path"):
str(tmp_path)}):
@@ -1693,6 +1700,7 @@ class TestGitDagBundle:
"""Test that InvalidGitRepositoryError after retry is re-raised
(wrapped in AirflowException by caller)."""
mock_githook.return_value.repo_url =
"[email protected]:apache/airflow.git"
mock_githook.return_value.env = {}
+ mock_githook.return_value.uses_github_app_auth = False
# Set up exists to return True for the bare repo path
mock_exists.return_value = True
@@ -1714,6 +1722,42 @@ class TestGitDagBundle:
# Verify Repo was called twice (failed attempt + failed retry)
assert mock_repo_class.call_count == 2
+ @pytest.mark.parametrize(
+ ("github_app_auth", "expected_attempts"),
+ [
+ pytest.param(True, 5, id="github-app-token-backs-off"),
+ pytest.param(False, 2, id="other-auth-fails-fast"),
+ ],
+ )
+ @mock.patch("airflow.providers.git.bundles.git.GitHook")
+ def test_clone_bare_repo_retry_policy_depends_on_auth(
+ self, mock_githook, github_app_auth, expected_attempts
+ ):
+ """GitHub rejects a just-issued App installation token with
"Repository not found" for a few
+ seconds, so only a GitHub App connection gets extra attempts with a
wait in between. Every other
+ connection keeps failing fast: a repository it cannot clone now will
not appear a few seconds later."""
+ mock_githook.return_value.repo_url = AIRFLOW_HTTPS_URL
+ mock_githook.return_value.env = {}
+ mock_githook.return_value.uses_github_app_auth = github_app_auth
+ bundle = GitDagBundle(name="test", git_conn_id=CONN_HTTPS,
tracking_ref=GIT_DEFAULT_BRANCH)
+
+ sleeps = []
+ with (
+ mock.patch(
+ "airflow.providers.git.bundles.git.Repo.clone_from",
+ side_effect=GitCommandError(["git", "clone"], 128,
stderr="remote: Repository not found."),
+ ) as mock_clone,
+ mock.patch.object(GitDagBundle._clone_bare_repo_if_required.retry,
"sleep", sleeps.append),
+ pytest.raises(GitCommandError),
+ ):
+ bundle._clone_bare_repo_if_required()
+
+ assert mock_clone.call_count == expected_attempts
+ if github_app_auth:
+ assert all(s > 0 for s in sleeps), "each retry must wait for the
token to become usable"
+ else:
+ assert not any(sleeps), "no wait for connections the token window
cannot affect"
+
@mock.patch("airflow.providers.git.bundles.git.GitHook")
def test_refresh_survives_upstream_tag_deletion(self, mock_githook,
git_repo):
"""Refresh succeeds when tracked tag is deleted upstream; local copy
persists."""
diff --git a/providers/git/tests/unit/git/hooks/test_git.py
b/providers/git/tests/unit/git/hooks/test_git.py
index 6a888852185..e014c8ae71b 100644
--- a/providers/git/tests/unit/git/hooks/test_git.py
+++ b/providers/git/tests/unit/git/hooks/test_git.py
@@ -821,6 +821,16 @@ class TestGitHook:
assert hook.github_installation_id == "67890"
assert hook.private_key is None
+ @pytest.mark.parametrize(
+ ("conn_id", "expected"),
+ [
+ pytest.param(CONN_APP_NO_KEY, True, id="github-app"),
+ pytest.param(CONN_HTTPS, False, id="token"),
+ ],
+ )
+ def test_uses_github_app_auth(self, conn_id, expected):
+ assert GitHook(git_conn_id=conn_id).uses_github_app_auth is expected
+
def test_app_auth_with_key_file_reads_file(self,
create_connection_without_db, tmp_path, monkeypatch):
key_file = tmp_path / "app_key.pem"
key_file.write_text("file_pem_key_content")
