This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ddf3c7a63e6 Breeze: fail fast when building provider sdists from a git
worktree (#65771)
ddf3c7a63e6 is described below
commit ddf3c7a63e60e773e0f10c20612e53aa12011eac
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Apr 25 00:24:26 2026 +0200
Breeze: fail fast when building provider sdists from a git worktree (#65771)
* Breeze: fail fast when building provider sdists from a git worktree
flit's --use-vcs silently produces incomplete sdists when run from a
`git worktree add ...` directory, because flit.vcs.identify_vcs() checks
`(p / ".git").is_dir()` and in a worktree `.git` is a file (gitdir:
pointer). flit then falls back to a minimal sdist that omits docs/,
tests/, provider.yaml and other tracked files, and the resulting
packages fail reproducibility checks against released sdists on
dist.apache.org — with no warning.
`breeze release-management prepare-provider-distributions` now exits 1
with a clear explanation and a workaround (use a plain checkout, or
pass `--distribution-format wheel` — wheels are unaffected) when it
detects it is running from a worktree and would build sdists.
Wheels and providers using hatchling with explicit sdist includes are
not affected, so only `sdist` and `both` formats trigger the check.
* Breeze: detect Docker-mounted worktrees in provider-sdist check
When Breeze runs prepare-provider-distributions from a git worktree, the
worktree's .git file carries an absolute gitdir: pointer to the main repo's
.git/worktrees/<name> directory. Only the worktree folder is bind-mounted into
Breeze's Docker container, so that pointer target is unreachable from inside
the build. flit's VCS detection then either fails or silently produces an
incomplete sdist.
Expand check_flit_worktree_compatibility to parse .git and branch on the
failure mode: read error, unexpected format, missing gitdir target (Docker
mount case), or healthy host-side worktree. Tests cover all paths.
* Link upstream flit issue/PR + tracking issue at the workaround site
Add references to pypa/flit#798 and pypa/flit#799 in the docstring of
check_flit_worktree_compatibility, and the Airflow tracking issue
apache/airflow#65772 at both the helper and its call site. CLAUDE.md requires
the tracking-issue URL to appear at the workaround site in the code so the
follow-up work is discoverable from any grep or code review.
---
.../commands/release_management_commands.py | 6 +
.../prepare_providers/provider_distributions.py | 109 +++++++++++++++++-
dev/breeze/tests/test_provider_distributions.py | 127 +++++++++++++++++++++
3 files changed, 241 insertions(+), 1 deletion(-)
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 58f4e6c4ad4..5104df663fe 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -112,6 +112,7 @@ from
airflow_breeze.prepare_providers.provider_distributions import (
PrepareReleasePackageTagExistException,
PrepareReleasePackageWrongSetupException,
build_provider_distribution,
+ check_flit_worktree_compatibility,
cleanup_build_remnants,
get_packages_list_to_act_on,
move_built_distributions_and_cleanup,
@@ -1157,6 +1158,11 @@ def prepare_provider_distributions(
version_suffix: str,
):
perform_environment_checks()
+ # Workaround for pypa/flit#798 (fix PR pypa/flit#799) plus the Breeze
+ # Docker-mount case. Remove this call and the helper it invokes once
+ # the conditions in the tracking issue are met:
+ # https://github.com/apache/airflow/issues/65772
+ check_flit_worktree_compatibility(distribution_format)
fix_ownership_using_docker()
cleanup_python_generated_files()
console_print("\n[info]Cleaning generated _api folders from docs
directories")
diff --git
a/dev/breeze/src/airflow_breeze/prepare_providers/provider_distributions.py
b/dev/breeze/src/airflow_breeze/prepare_providers/provider_distributions.py
index ca3f02e3e20..181366fdaee 100644
--- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_distributions.py
+++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_distributions.py
@@ -33,11 +33,118 @@ from airflow_breeze.utils.packages import (
get_removed_provider_ids,
tag_exists_for_provider,
)
-from airflow_breeze.utils.path_utils import AIRFLOW_DIST_PATH
+from airflow_breeze.utils.path_utils import AIRFLOW_DIST_PATH,
AIRFLOW_ROOT_PATH
from airflow_breeze.utils.run_utils import run_command
from airflow_breeze.utils.version_utils import is_local_package_version
+def check_flit_worktree_compatibility(distribution_format: str) -> None:
+ """Refuse to build provider sdists when VCS detection will silently break.
+
+ Provider sdists are built with ``flit --use-vcs``. flit identifies the VCS
+ via ``flit.vcs.identify_vcs``, which checks ``(p / ".git").is_dir()``.
+ That check fails in two related scenarios, both of which produce a
+ broken sdist with no warning from flit:
+
+ 1. **Plain git worktree on the host.** Inside a ``git worktree add ...``
+ directory ``.git`` is a regular file containing a ``gitdir: <path>``
+ pointer, not a directory. ``is_dir()`` returns False, ``identify_vcs``
+ returns ``None``, and flit falls back to a non-VCS sdist that omits
+ tracked files like ``docs/``, ``tests/`` and ``provider.yaml``. The
+ resulting packages fail reproducibility checks against the released
+ sdists on dist.apache.org.
+ 2. **Breeze Docker builds with a worktree mounted.** When the build runs
+ inside Breeze's container, only the worktree directory is
+ bind-mounted. The absolute ``gitdir:`` path baked into ``.git``
+ points somewhere on the host (``<main_repo>/.git/worktrees/<name>``)
+ that does not exist inside the container, so even if flit recognised
+ worktrees, ``git ls-files`` would fail or misbehave.
+
+ Wheels are built from ``pyproject.toml`` metadata and are unaffected, so
+ this check is a no-op for ``--distribution-format wheel``. Providers
+ that use ``hatchling`` with an explicit
+ ``[tool.hatch.build.targets.sdist]`` ``include`` list are also
+ unaffected.
+
+ Upstream flit tracking:
+
+ - Issue: https://github.com/pypa/flit/issues/798
+ - Fix PR: https://github.com/pypa/flit/pull/799
+
+ Airflow-side follow-up (remove this workaround when both conditions in
+ the tracking issue are met):
+ https://github.com/apache/airflow/issues/65772
+ """
+ if distribution_format == "wheel":
+ return
+
+ git_marker = AIRFLOW_ROOT_PATH / ".git"
+ if not git_marker.is_file():
+ # Regular .git directory (or missing entirely) — flit's VCS detection
+ # works correctly, nothing to do.
+ return
+
+ # .git is a file — typical in a git worktree (or submodule). Parse the
+ # pointer so we can give a precise error for each failure mode, and so
+ # we detect the Breeze-in-Docker case where the target is unreachable.
+ try:
+ content = git_marker.read_text(encoding="utf-8").strip()
+ except OSError as err:
+ console_print(
+ f"\n[error]Cannot read {git_marker}: {err}.[/]\n"
+ "[info]Run this command from a plain `git checkout`, or pass "
+ "`--distribution-format wheel` (wheels are unaffected).[/]\n"
+ )
+ sys.exit(1)
+
+ if not content.startswith("gitdir:"):
+ console_print(
+ f"\n[error]Unexpected `.git` file format at {git_marker}.[/]\n\n"
+ f"[warning]Expected a `gitdir: <path>` pointer (git worktree or "
+ f"submodule), got:\n {content!r}[/]\n\n"
+ "[info]Run this command from a plain `git checkout`, or pass "
+ "`--distribution-format wheel` (wheels are unaffected).[/]\n"
+ )
+ sys.exit(1)
+
+ gitdir = Path(content[len("gitdir:") :].strip())
+ if not gitdir.exists():
+ console_print(
+ "\n[error]Cannot build provider sdists: git worktree pointer "
+ "targets a missing gitdir.[/]\n\n"
+ f"[warning]The `.git` file at `{git_marker}` points to "
+ f"`{gitdir}`, but that directory does not exist from here.[/]\n\n"
+ "[info]This typically happens when Breeze runs the build inside "
+ "Docker: only the worktree directory is bind-mounted into the "
+ "container, so the main repo's `.git/worktrees/<name>` folder "
+ "that the pointer references is not reachable. flit then either "
+ "fails outright or silently produces an incomplete sdist that "
+ "omits tracked files.[/]\n\n"
+ "[info]Run this command from a plain `git checkout` (not a "
+ "`git worktree add ...` directory), or pass "
+ "`--distribution-format wheel` — wheels don't need VCS metadata "
+ "and are unaffected.[/]\n"
+ )
+ sys.exit(1)
+
+ # Healthy worktree on the host. flit's `.is_dir()` check still misses it,
+ # so we refuse sdist builds here too.
+ console_print(
+ "\n[error]Cannot build provider sdists from a git worktree.[/]\n\n"
+ "[warning]flit's `--use-vcs` does not recognise git worktrees "
+ "(flit.vcs.identify_vcs uses `.git.is_dir()`, but in a worktree "
+ "`.git` is a file). flit silently falls back to a minimal sdist "
+ "that omits docs/, tests/, provider.yaml and other tracked files, "
+ "so the resulting packages fail reproducibility checks against "
+ "the released sdists on dist.apache.org.[/]\n\n"
+ "[info]Run this command from a plain `git checkout` (not a "
+ "`git worktree add ...` directory). If you only need wheels, pass "
+ "`--distribution-format wheel` — wheels are built from "
+ "`pyproject.toml` metadata and are not affected by this bug.[/]\n"
+ )
+ sys.exit(1)
+
+
class PrepareReleasePackageTagExistException(Exception):
"""Tag already exist for the package."""
diff --git a/dev/breeze/tests/test_provider_distributions.py
b/dev/breeze/tests/test_provider_distributions.py
new file mode 100644
index 00000000000..96ef113471c
--- /dev/null
+++ b/dev/breeze/tests/test_provider_distributions.py
@@ -0,0 +1,127 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from airflow_breeze.prepare_providers import provider_distributions
+from airflow_breeze.prepare_providers.provider_distributions import (
+ check_flit_worktree_compatibility,
+)
+
+# Rich output is wrapped to the terminal width and peppered with ANSI escapes;
+# strip both when asserting on message contents.
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
+
+
+def _plain(out: str, err: str) -> str:
+ return " ".join(_ANSI_RE.sub("", out + err).split())
+
+
[email protected]
+def healthy_worktree_root(tmp_path: Path) -> Path:
+ """A worktree whose `gitdir:` pointer targets a real directory."""
+ gitdir = tmp_path / "main_repo" / ".git" / "worktrees" / "feature"
+ gitdir.mkdir(parents=True)
+ worktree = tmp_path / "worktree"
+ worktree.mkdir()
+ (worktree / ".git").write_text(f"gitdir: {gitdir}\n")
+ return worktree
+
+
[email protected]
+def broken_worktree_root(tmp_path: Path) -> Path:
+ """A worktree whose `gitdir:` pointer targets a path that does not exist.
+
+ This is the Breeze-in-Docker shape: the worktree directory is mounted
+ into the container but the absolute `gitdir:` path is not.
+ """
+ worktree = tmp_path / "worktree"
+ worktree.mkdir()
+ (worktree / ".git").write_text("gitdir:
/nonexistent/main/.git/worktrees/feature\n")
+ return worktree
+
+
[email protected]
+def malformed_git_file_root(tmp_path: Path) -> Path:
+ """`.git` is a file but the content is not a `gitdir:` pointer."""
+ (tmp_path / ".git").write_text("this is not a git pointer\n")
+ return tmp_path
+
+
[email protected]
+def plain_checkout_root(tmp_path: Path) -> Path:
+ (tmp_path / ".git").mkdir()
+ return tmp_path
+
+
[email protected]("dist_format", ["sdist", "both"])
+def test_healthy_worktree_blocks_sdist_build(healthy_worktree_root: Path,
dist_format: str) -> None:
+ with patch.object(provider_distributions, "AIRFLOW_ROOT_PATH",
healthy_worktree_root):
+ with pytest.raises(SystemExit) as exc_info:
+ check_flit_worktree_compatibility(dist_format)
+ assert exc_info.value.code == 1
+
+
+def test_healthy_worktree_allows_wheel_build(healthy_worktree_root: Path) ->
None:
+ with patch.object(provider_distributions, "AIRFLOW_ROOT_PATH",
healthy_worktree_root):
+ check_flit_worktree_compatibility("wheel")
+
+
[email protected]("dist_format", ["sdist", "both"])
+def test_broken_gitdir_pointer_exits_with_docker_hint(
+ broken_worktree_root: Path, capsys: pytest.CaptureFixture[str],
dist_format: str
+) -> None:
+ with patch.object(provider_distributions, "AIRFLOW_ROOT_PATH",
broken_worktree_root):
+ with pytest.raises(SystemExit) as exc_info:
+ check_flit_worktree_compatibility(dist_format)
+ assert exc_info.value.code == 1
+ # The Docker-specific message must surface — the broken pointer is the
+ # signal we use to tell the user that this is the container-mount case.
+ captured = capsys.readouterr()
+ output = _plain(captured.out, captured.err)
+ assert "Docker" in output
+ assert "does not exist" in output
+
+
+def test_broken_gitdir_still_allows_wheel_build(broken_worktree_root: Path) ->
None:
+ with patch.object(provider_distributions, "AIRFLOW_ROOT_PATH",
broken_worktree_root):
+ check_flit_worktree_compatibility("wheel")
+
+
[email protected]("dist_format", ["sdist", "both"])
+def test_malformed_git_file_exits(malformed_git_file_root: Path, dist_format:
str) -> None:
+ with patch.object(provider_distributions, "AIRFLOW_ROOT_PATH",
malformed_git_file_root):
+ with pytest.raises(SystemExit) as exc_info:
+ check_flit_worktree_compatibility(dist_format)
+ assert exc_info.value.code == 1
+
+
+def test_malformed_git_file_allows_wheel_build(malformed_git_file_root: Path)
-> None:
+ with patch.object(provider_distributions, "AIRFLOW_ROOT_PATH",
malformed_git_file_root):
+ check_flit_worktree_compatibility("wheel")
+
+
+def test_plain_checkout_allows_all_formats(plain_checkout_root: Path) -> None:
+ with patch.object(provider_distributions, "AIRFLOW_ROOT_PATH",
plain_checkout_root):
+ check_flit_worktree_compatibility("sdist")
+ check_flit_worktree_compatibility("both")
+ check_flit_worktree_compatibility("wheel")
