This is an automated email from the ASF dual-hosted git repository.
ephraimbuddy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ea289c5bb41 Add the artifact store for manifest-backed versioned Dag
bundles (#75498)
ea289c5bb41 is described below
commit ea289c5bb41a6fa206f23dac210055547ea1bc7d
Author: Ephraim Anierobi <[email protected]>
AuthorDate: Thu Nov 19 08:20:30 2026 +0100
Add the artifact store for manifest-backed versioned Dag bundles (#75498)
* Add the artifact store for manifest-backed versioned Dag bundles
Manifest-backed versioned Dag bundles need durable published artifacts
shared by publishers and workers without tying bundle lifecycle logic
to a particular storage backend.
* Test that fetched snapshot directories stay writable
Fetch copies files without directory metadata, so the copied
directories do not take the snapshot's read-only mode.
---
.../airflow/dag_processing/bundles/_filesystem.py | 175 +++
.../dag_processing/bundles/artifact_store.py | 202 +++
.../bundles/filesystem_artifact_store.py | 552 ++++++++
.../src/airflow/dag_processing/bundles/manifest.py | 14 +-
.../dag_processing/bundles/test__filesystem.py | 94 ++
.../bundles/test_filesystem_artifact_store.py | 1407 ++++++++++++++++++++
6 files changed, 2438 insertions(+), 6 deletions(-)
diff --git a/airflow-core/src/airflow/dag_processing/bundles/_filesystem.py
b/airflow-core/src/airflow/dag_processing/bundles/_filesystem.py
new file mode 100644
index 00000000000..1ba487d75d8
--- /dev/null
+++ b/airflow-core/src/airflow/dag_processing/bundles/_filesystem.py
@@ -0,0 +1,175 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Filesystem helpers shared by the filesystem artifact store and the manifest
bundle's local cache.
+
+Not an extension point: code outside ``airflow.dag_processing.bundles`` must
reach published
+artifacts through ``ArtifactStore``.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import shutil
+import stat
+from pathlib import Path
+from typing import Any, NoReturn
+
+from airflow.dag_processing.bundles.artifact_store import
BundleManifestNotFoundError
+from airflow.dag_processing.bundles.manifest import (
+ SHA256_VERSION_PREFIX,
+ BundleManifestError,
+ is_sha256_hex,
+)
+
+log = logging.getLogger(__name__)
+
+
+def read_json_file(path: Path, *, missing_message: str, invalid_message: str)
-> dict[str, Any]:
+ """Read a JSON object from a published document, raising
``missing_message`` if it is absent."""
+ try:
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ except FileNotFoundError as e:
+ raise BundleManifestNotFoundError(missing_message) from e
+ except (UnicodeDecodeError, json.JSONDecodeError) as e:
+ raise BundleManifestError(invalid_message) from e
+ except OSError as e:
+ raise BundleManifestError(f"Cannot read {path}: {e}") from e
+
+ if not isinstance(payload, dict):
+ raise BundleManifestError(invalid_message)
+ return payload
+
+
+def get_path_mode(path: Path) -> int | None:
+ """
+ Return the ``st_mode`` of ``path``, or ``None`` if it does not exist.
+
+ Use instead of ``Path.exists()`` or ``Path.is_dir()`` on published
artifacts: from
+ Python 3.14 those return False on any ``OSError``, so a permission or I/O
failure
+ would read as a missing artifact.
+ """
+ try:
+ return path.stat().st_mode
+ except FileNotFoundError:
+ return None
+
+
+def raise_walk_error(error: OSError) -> NoReturn:
+ """``os.walk`` ``onerror`` handler; by default ``os.walk`` skips
directories it cannot list."""
+ raise error
+
+
+def ensure_destination_is_within_root(root: Path, destination: Path) -> None:
+ try:
+ destination.resolve().relative_to(root.resolve())
+ except ValueError:
+ raise BundleManifestError(
+ f"Bundle manifest path {destination} resolves outside snapshot
root {root}"
+ ) from None
+
+
+def _make_tree_writable(path: Path) -> None:
+ directories: list[Path] = []
+ for dirpath, _, filenames in os.walk(path):
+ directory = Path(dirpath)
+ directories.append(directory)
+ for filename in filenames:
+ child = directory / filename
+ if child.is_symlink():
+ continue
+ try:
+ child.chmod(stat.S_IMODE(child.stat().st_mode) | 0o200)
+ except OSError:
+ continue
+
+ for directory in sorted(directories, key=lambda path: len(path.parts),
reverse=True):
+ try:
+ directory.chmod(stat.S_IMODE(directory.stat().st_mode) | 0o700)
+ except OSError:
+ continue
+
+
+def remove_bundle_tree_forcefully(path: Path) -> None:
+ """Remove a bundle tree, adding write permissions first when a read-only
snapshot blocks removal."""
+ if path.is_symlink() or not path.is_dir():
+ path.unlink(missing_ok=True)
+ return
+ try:
+ shutil.rmtree(path)
+ except OSError:
+ _make_tree_writable(path)
+ shutil.rmtree(path)
+
+
+def get_temp_snapshot_prefix(version: str) -> str:
+ return f".{version}."
+
+
+def remove_orphaned_temp_snapshots(versions_dir: Path) -> None:
+ if not versions_dir.exists():
+ return
+
+ removed_orphans = False
+ for path in versions_dir.iterdir():
+ if not _is_temp_snapshot_dir(path):
+ continue
+ try:
+ remove_bundle_tree_forcefully(path)
+ except OSError:
+ # One undeletable orphan (e.g. owned by another OS user) must not
block
+ # materialization of valid versions.
+ log.warning("Could not remove orphaned temp snapshot %s", path,
exc_info=True)
+ continue
+ removed_orphans = True
+
+ if removed_orphans:
+ fsync_directory(versions_dir)
+
+
+def _is_temp_snapshot_dir(path: Path) -> bool:
+ if path.is_symlink() or not path.is_dir():
+ return False
+
+ name = path.name
+ if not name.startswith(f".{SHA256_VERSION_PREFIX}"):
+ return False
+
+ digest, separator, suffix =
name.removeprefix(f".{SHA256_VERSION_PREFIX}").partition(".")
+ return separator == "." and bool(suffix) and is_sha256_hex(digest)
+
+
+def fsync_file(path: Path) -> None:
+ with path.open("rb") as file:
+ os.fsync(file.fileno())
+
+
+def fsync_directory(path: Path) -> None:
+ flags = getattr(os, "O_DIRECTORY", 0)
+ fd = os.open(path, os.O_RDONLY | flags)
+ try:
+ os.fsync(fd)
+ finally:
+ os.close(fd)
+
+
+def fsync_tree_directories(root: Path) -> None:
+ directories = [Path(dirpath) for dirpath, _, _ in os.walk(root,
onerror=raise_walk_error)]
+ for directory in sorted(directories, key=lambda path: len(path.parts),
reverse=True):
+ fsync_directory(directory)
diff --git a/airflow-core/src/airflow/dag_processing/bundles/artifact_store.py
b/airflow-core/src/airflow/dag_processing/bundles/artifact_store.py
new file mode 100644
index 00000000000..1d90b98ac8a
--- /dev/null
+++ b/airflow-core/src/airflow/dag_processing/bundles/artifact_store.py
@@ -0,0 +1,202 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Contract for the published-artifact backend of manifest-backed Dag bundles.
+
+An artifact store holds a bundle's release reference, the one mutable document
naming the
+current version, and one immutable snapshot per published version: the bundle
files and the
+manifest that lists them. Manifest-backed Dag bundles access published
artifacts only through
+this interface.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from airflow.dag_processing.bundles.manifest import BundleManifestError
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+ from contextlib import AbstractContextManager
+ from pathlib import Path
+
+
+class BundleManifestNotFoundError(BundleManifestError, FileNotFoundError):
+ """
+ Raised when a published artifact -- the release reference or a snapshot --
does not exist.
+
+ It is a ``FileNotFoundError`` so that code which already handles a missing
file treats a
+ missing artifact the same way, and a ``BundleManifestError`` so that it is
still reported
+ as a bundle error. It is distinct from
``BundleManifestSourceChangedError``: an artifact
+ that is absent is not a source that changed while it was read.
+ """
+
+
+class BundleManifestRefDurabilityError(BundleManifestError):
+ """
+ Raised when the release reference was, or may have been, replaced but its
durability is unconfirmed.
+
+ ``ref_locator`` identifies the reference to inspect before retrying
publication. ``version``
+ is the attempted version, or ``None`` when the payload has no string
version. The replacement
+ may already be visible to readers. The underlying storage error is chained.
+ """
+
+ def __init__(self, *, ref_locator: str, version: str | None) -> None:
+ self.ref_locator = ref_locator
+ self.version = version
+ super().__init__(
+ f"Release reference at {ref_locator} may have been replaced, but
durability could not be confirmed. "
+ "Check the current reference before retrying publication."
+ )
+
+
+class ArtifactStoreConflictError(BundleManifestError):
+ """
+ Raised by a compare-and-swap write, such as ``write_ref``, when another
publisher wrote first.
+
+ It is retryable: the caller re-reads the reference and decides again.
+ """
+
+
+@dataclass(frozen=True)
+class PublishedSnapshot:
+ """
+ Result of ``ArtifactStore.publish_snapshot``.
+
+ ``manifest`` is the manifest of the committed snapshot: for a version that
was already
+ published, the existing one. ``created`` is whether this call committed
the snapshot.
+ """
+
+ manifest: dict[str, Any]
+ created: bool
+
+
+class ArtifactStore(ABC):
+ """
+ Storage backend for a bundle's published release reference and snapshots.
+
+ Storage failures raise ``BundleManifestError``, or
``BundleManifestNotFoundError`` when an
+ artifact is missing. ``BundleManifestSourceChangedError`` is reserved for
reads of the
+ publish source.
+ """
+
+ bundle_name: str
+
+ @property
+ @abstractmethod
+ def backend_type(self) -> str:
+ """``backend.type`` recorded in the manifests and release references
this store publishes."""
+
+ @property
+ @abstractmethod
+ def ref_locator(self) -> str:
+ """Opaque identifier of the release reference, for messages and
results."""
+
+ @abstractmethod
+ def snapshot_locator(self, version: str) -> str:
+ """Opaque identifier of one published snapshot, for messages and
results."""
+
+ @abstractmethod
+ def read_ref(self) -> dict[str, Any]:
+ """
+ Read the release reference payload.
+
+ Raises ``BundleManifestNotFoundError`` when no reference is published,
and
+ ``BundleManifestError`` when it is not a JSON object.
+ """
+
+ @abstractmethod
+ def write_ref(self, payload: dict[str, Any]) -> None:
+ """
+ Atomically replace the release reference. Must be called inside
``publication_guard``.
+
+ Returns only once the replacement is durable. Raises
``BundleManifestRefDurabilityError``
+ when the replacement completed or may have completed but its
durability is unconfirmed;
+ the replacement must not be rolled back. Other failures raise
``BundleManifestError``.
+
+ When ``publication_guard`` is not a lock, the replacement is
conditional on the reference
+ read inside the same guard being unchanged, or still absent if that
read found none; with
+ no such read, on the reference read just before writing. Raises
+ ``ArtifactStoreConflictError`` when another publisher replaced it
first, unless the
+ winner wrote byte-identical content.
+ """
+
+ @abstractmethod
+ def publication_guard(self) -> AbstractContextManager[None]:
+ """
+ Protect a publisher's read-check-write section against concurrent
publishers.
+
+ Either a lock that serializes the bundle's publishers across hosts
and, on acquire,
+ removes leftovers of earlier failed publications; or, for a store that
writes no
+ temporary artifacts, the scope of the conditions ``write_ref``
applies. Entering such a
+ guard discards conditions from reads made before it.
+ """
+
+ @abstractmethod
+ def prepare_publish_areas(self) -> None:
+ """Create the snapshot and reference areas publishers write to."""
+
+ @abstractmethod
+ def snapshot_exists(self, version: str) -> bool:
+ """Whether a committed snapshot exists for the version."""
+
+ @abstractmethod
+ def fetch_snapshot(
+ self,
+ version: str,
+ destination: Path,
+ *,
+ structural_validator: Callable[[Path], None],
+ ) -> None:
+ """
+ Transfer a published snapshot into ``destination``, which must be
absent or empty.
+
+ ``destination`` receives only regular files: the files the manifest
lists, with their
+ executable bits, and the manifest at ``MANIFEST_FILE_NAME``. Each
manifest path must pass
+ ``validate_bundle_relative_path`` before anything is written to it. A
store that can
+ access the published tree must call ``structural_validator`` on it
before transferring
+ anything; the validator checks those paths and rejects any other
entry. Raises
+ ``BundleManifestNotFoundError`` when the version is not published, and
``ValueError``
+ when ``destination`` is not empty.
+ """
+
+ @abstractmethod
+ def publish_snapshot(
+ self,
+ *,
+ manifest: dict[str, Any],
+ source_root: Path,
+ validate_existing: Callable[[Path], None],
+ ) -> PublishedSnapshot:
+ """
+ Commit an immutable snapshot of ``manifest`` from the prepared tree at
``source_root``.
+
+ The snapshot is published under ``manifest["version"]`` and is never
visible
+ half-written. Must be called inside ``publication_guard``.
+
+ When the version is already committed, the existing snapshot is kept
and returned:
+ ``validate_existing`` checks it against its version and bundle, or,
without a local tree
+ to pass, the store checks its manifest against the version. That
manifest may differ
+ from ``manifest`` in fields outside the version, such as per-file
object version ids.
+ When ``publication_guard`` is not a lock, the manifest is committed
only if absent, so
+ a concurrent publisher of the same version cannot overwrite it.
+
+ Returns the committed manifest and whether this call created the
snapshot, once the
+ snapshot is durable, including one committed by an earlier failed
attempt.
+ """
diff --git
a/airflow-core/src/airflow/dag_processing/bundles/filesystem_artifact_store.py
b/airflow-core/src/airflow/dag_processing/bundles/filesystem_artifact_store.py
new file mode 100644
index 00000000000..7a10284054e
--- /dev/null
+++
b/airflow-core/src/airflow/dag_processing/bundles/filesystem_artifact_store.py
@@ -0,0 +1,552 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Shared-filesystem artifact store for manifest-backed Dag bundles."""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import stat
+import tempfile
+import time
+from contextlib import contextmanager, suppress
+from fcntl import LOCK_EX, LOCK_NB, LOCK_UN, flock
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from airflow.dag_processing.bundles._filesystem import (
+ ensure_destination_is_within_root,
+ fsync_directory,
+ fsync_file,
+ fsync_tree_directories,
+ get_path_mode,
+ get_temp_snapshot_prefix,
+ raise_walk_error,
+ read_json_file,
+ remove_bundle_tree_forcefully,
+ remove_orphaned_temp_snapshots,
+)
+from airflow.dag_processing.bundles.artifact_store import (
+ ArtifactStore,
+ BundleManifestNotFoundError,
+ BundleManifestRefDurabilityError,
+ PublishedSnapshot,
+)
+from airflow.dag_processing.bundles.manifest import (
+ MANIFEST_FILE_NAME,
+ BundleManifestError,
+ BundleManifestSourceChangedError,
+ compute_file_sha256,
+ raise_source_access_error,
+ serialize_bundle_version_manifest,
+ validate_bundle_relative_path,
+)
+
+if TYPE_CHECKING:
+ from collections.abc import Callable, Iterator
+
+log = logging.getLogger(__name__)
+
+DEFAULT_PUBLICATION_LOCK_TIMEOUT = 600.0
+_LOCK_POLL_INTERVAL = 1.0
+_LOCK_WAIT_LOG_INTERVAL = 30.0
+
+
+class FilesystemArtifactStore(ArtifactStore):
+ """
+ Artifact store for a ``published_root`` on a filesystem shared by
publishers and Airflow.
+
+ Layout under ``published_root``: ``refs/<bundle>/latest.json`` (release
reference),
+ ``versions/<bundle>/<version>/`` (snapshots) and ``_locks/<bundle>.lock``
(publication lock).
+ ``published_root`` must already exist; the store creates only the
directories below it.
+ Provision it as a dedicated directory inside the shared filesystem so that
a missing
+ mount also makes it disappear. An existing root, including the mount point
itself,
+ does not prove the filesystem is mounted; deployments must ensure mount
availability.
+
+ Publishers need write access to the publication directories and read/write
access to
+ the lock file. For different OS users, give ``published_root`` a shared
publisher group
+ with group write and the setgid bit: new directories use those bits from
``published_root``.
+
+ Publishers on different hosts serialize through ``flock`` on the lock
file, so the
+ filesystem must enforce ``flock`` across hosts: NFSv4 does, NFSv3 only
with the lock
+ manager (lockd) running, and FUSE or object-store mounts often accept
``flock`` without
+ enforcing it, which leaves concurrent publishers unserialized.
+
+ Airflow never deletes published snapshots: a Dag run can request any
version it pinned,
+ for example on a retry, rerun or callback. Snapshots are read-only and may
be owned by
+ the publisher's OS user, so to prune one that no run can still request,
remove it as
+ that user after adding the owner write bit (``chmod -R u+w``).
+
+ :param lock_timeout: Seconds to wait for another publisher to release the
lock before
+ failing. The lock is held while a whole snapshot is materialized, so
allow for the
+ largest bundle on the slowest shared filesystem.
+ """
+
+ backend_type = "local"
+
+ def __init__(
+ self,
+ *,
+ bundle_name: str,
+ published_root: Path,
+ lock_timeout: float = DEFAULT_PUBLICATION_LOCK_TIMEOUT,
+ ) -> None:
+ self.bundle_name = bundle_name
+ self._root = published_root
+ self._ref_path = published_root / "refs" / bundle_name / "latest.json"
+ self._snapshots_root = published_root / "versions" / bundle_name
+ self._lock_path = published_root / "_locks" / f"{bundle_name}.lock"
+ self._lock_timeout = lock_timeout
+ self._synced_dirs: set[Path] = set()
+
+ @property
+ def published_root(self) -> Path:
+ """The ``published_root`` directory, for checks that need a filesystem
path."""
+ return self._root
+
+ @property
+ def ref_locator(self) -> str:
+ return str(self._ref_path)
+
+ def snapshot_locator(self, version: str) -> str:
+ return str(self._snapshots_root / version)
+
+ def read_ref(self) -> dict[str, Any]:
+ return read_json_file(
+ self._ref_path,
+ missing_message=f"Bundle '{self.bundle_name}' has no release
reference at {self._ref_path}",
+ invalid_message=(
+ f"Bundle '{self.bundle_name}' release reference
{self._ref_path} is not a JSON object"
+ ),
+ )
+
+ def write_ref(self, payload: dict[str, Any]) -> None:
+ with _raise_storage_error(f"write release reference {self._ref_path}"):
+ self._ensure_publication_dir(self._ref_path.parent)
+ _write_json_atomically(self._ref_path, payload)
+
+ @contextmanager
+ def publication_guard(self):
+ """Serialize publishers through a lock stored with the authoritative
snapshots."""
+ with _raise_storage_error(f"open publication lock {self._lock_path}"):
+ self._ensure_publication_dir(self._lock_path.parent)
+ # Linux NFS implements flock with fcntl locks, which require a
writable
+ # descriptor for an exclusive lock.
+ try:
+ fd = os.open(self._lock_path, os.O_CREAT | os.O_RDWR, 0o664)
+ except PermissionError as e:
+ raise BundleManifestError(
+ f"Cannot open publication lock {self._lock_path}:
publishers need read/write "
+ "access. Set its group to a shared publisher group and
grant group read/write "
+ "access (chmod 0664), and ensure the lock directory is
writable."
+ ) from e
+ with os.fdopen(fd, "r+b") as lock_file:
+ # os.open masks the mode with the umask; repair it whenever this
publisher
+ # owns the file so other members of the publisher group can
acquire it.
+ with _raise_storage_error(f"set permissions on publication lock
{self._lock_path}"):
+ try:
+ os.fchmod(fd, stat.S_IMODE(os.fstat(fd).st_mode) | 0o664)
+ except OSError:
+ if os.fstat(fd).st_uid == os.geteuid():
+ raise
+ self._acquire_publication_lock(lock_file)
+ published = False
+ try:
+ with _raise_storage_error(f"remove leftovers of failed
publications under {self._root}"):
+ self._remove_publish_leftovers()
+ yield
+ published = True
+ finally:
+ # Unlock explicitly, not only through close(): a forked child
sharing this open
+ # file description would keep holding the lock after this
process closes it.
+ try:
+ flock(lock_file, LOCK_UN)
+ except OSError as e:
+ if published:
+ raise BundleManifestError(
+ f"Cannot release publication lock
{self._lock_path}: {e}"
+ ) from e
+ # The publication's own error tells callers how to
recover; keep it.
+ log.warning("Cannot release publication lock %s",
self._lock_path, exc_info=True)
+
+ def _acquire_publication_lock(self, lock_file) -> None:
+ deadline = time.monotonic() + self._lock_timeout
+ next_log_time = time.monotonic()
+ while True:
+ try:
+ flock(lock_file, LOCK_EX | LOCK_NB)
+ except BlockingIOError:
+ pass
+ except OSError as e:
+ raise BundleManifestError(
+ f"Cannot lock publication lock {self._lock_path}: {e}. The
filesystem under "
+ "published_root must support flock across hosts; NFSv3
needs lockd."
+ ) from e
+ else:
+ return
+ now = time.monotonic()
+ if now >= deadline:
+ raise BundleManifestError(
+ f"Timed out after {self._lock_timeout:g}s waiting for
publication lock "
+ f"{self._lock_path}; another publisher still holds it."
+ )
+ if now >= next_log_time:
+ log.info("Waiting for publication lock %s held by another
publisher", self._lock_path)
+ next_log_time = now + _LOCK_WAIT_LOG_INTERVAL
+ time.sleep(min(_LOCK_POLL_INTERVAL, deadline - now))
+
+ def prepare_publish_areas(self) -> None:
+ with _raise_storage_error(f"create publication directories under
{self._root}"):
+ self._ensure_publication_dir(self._snapshots_root)
+ self._ensure_publication_dir(self._ref_path.parent)
+
+ def snapshot_exists(self, version: str) -> bool:
+ with _raise_storage_error(f"check published snapshot
{self._snapshots_root / version}"):
+ mode = get_path_mode(self._snapshots_root / version)
+ return mode is not None and stat.S_ISDIR(mode)
+
+ def fetch_snapshot(
+ self,
+ version: str,
+ destination: Path,
+ *,
+ structural_validator: Callable[[Path], None],
+ ) -> None:
+ published_version_path = self._snapshots_root / version
+ with _raise_storage_error(f"fetch published snapshot
{published_version_path}"):
+ published_mode = get_path_mode(published_version_path)
+ if published_mode is None or not stat.S_ISDIR(published_mode):
+ raise BundleManifestNotFoundError(
+ f"Bundle '{self.bundle_name}' version '{version}' is not
published at "
+ f"{published_version_path}. Publish or restore the
immutable snapshot before "
+ "running pinned work."
+ )
+ if destination.exists() and any(destination.iterdir()):
+ raise ValueError(f"Snapshot fetch destination {destination}
must be absent or empty")
+ structural_validator(published_version_path)
+ _copy_tree_contents(published_version_path, destination)
+
+ def publish_snapshot(
+ self,
+ *,
+ manifest: dict[str, Any],
+ source_root: Path,
+ validate_existing: Callable[[Path], None],
+ ) -> PublishedSnapshot:
+ version_path = self._snapshots_root / manifest["version"]
+ with _raise_storage_error(f"publish snapshot {version_path}"):
+ version_mode = get_path_mode(version_path)
+ if version_mode is None:
+ created = _materialize_manifest_snapshot(
+ source_path=source_root,
+ manifest=manifest,
+ versions_dir=self._snapshots_root,
+ version_path=version_path,
+ )
+ elif not stat.S_ISDIR(version_mode):
+ raise BundleManifestError(
+ f"Bundle snapshot path exists but is not a directory:
{version_path}"
+ )
+ else:
+ created = False
+ committed_manifest = manifest
+ if not created:
+ validate_existing(version_path)
+ manifest_file = version_path / MANIFEST_FILE_NAME
+ committed_manifest = read_json_file(
+ manifest_file,
+ missing_message=f"Bundle snapshot {version_path} has no
manifest",
+ invalid_message=f"Bundle snapshot manifest {manifest_file}
is not a JSON object",
+ )
+ # An earlier attempt may have committed the snapshot and failed
before this sync.
+ fsync_directory(self._snapshots_root)
+ return PublishedSnapshot(manifest=committed_manifest,
created=created)
+
+ def _remove_publish_leftovers(self) -> None:
+ remove_orphaned_temp_snapshots(self._snapshots_root)
+ _remove_orphaned_atomic_temp_files(self._ref_path)
+
+ def _ensure_publication_dir(self, path: Path) -> None:
+ """Create a publication directory, syncing existing ones only on this
store's first call for it."""
+ _ensure_public_dir(path, root=self._root, sync_existing=path not in
self._synced_dirs)
+ self._synced_dirs.add(path)
+
+
+@contextmanager
+def _raise_storage_error(action: str) -> Iterator[None]:
+ """
+ Re-raise an ``OSError`` from published-artifact storage as a bundle error.
+
+ A missing artifact becomes ``BundleManifestNotFoundError``; any other
failure is a
+ ``BundleManifestError``, never ``BundleManifestSourceChangedError``, which
publishers
+ treat as a benign concurrent edit of the source and retry quietly.
+ """
+ try:
+ yield
+ except BundleManifestError:
+ raise
+ except FileNotFoundError as e:
+ raise BundleManifestNotFoundError(f"Cannot {action}: {e}") from e
+ except OSError as e:
+ raise BundleManifestError(f"Cannot {action}: {e}") from e
+
+
+def _copy_file_contents(source: Path, destination: Path) -> None:
+ """
+ Copy a file's bytes and permission bits, and none of its other metadata.
+
+ Not ``shutil.copy2``: its ``copystat`` carries over ACLs and extended
attributes, which the
+ chmods applied to snapshots and their cache copies do not clear.
+ """
+ shutil.copy(source, destination)
+
+
+def _copy_tree_contents(source_root: Path, destination: Path) -> None:
+ """Copy a tree with ``_copy_file_contents``; unlike ``shutil.copytree``,
copy no directory metadata."""
+ for dirpath, _, filenames in os.walk(source_root,
onerror=raise_walk_error):
+ directory = destination / Path(dirpath).relative_to(source_root)
+ directory.mkdir(parents=True, exist_ok=True)
+ for filename in filenames:
+ _copy_file_contents(Path(dirpath) / filename, directory / filename)
+
+
+def _materialize_manifest_snapshot(
+ *,
+ source_path: Path,
+ manifest: dict[str, Any],
+ versions_dir: Path,
+ version_path: Path,
+) -> bool:
+ """Commit the snapshot; return False if the rename failed but the version
exists and needs validation."""
+ version = manifest["version"]
+ tmp_path = Path(
+ tempfile.mkdtemp(
+ prefix=get_temp_snapshot_prefix(version),
+ dir=versions_dir,
+ )
+ )
+ try:
+ for file_info in manifest["files"]:
+ relative_path = validate_bundle_relative_path(file_info["path"])
+ source = source_path / relative_path
+ destination = tmp_path / relative_path
+ ensure_destination_is_within_root(tmp_path, destination)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ try:
+ _copy_file_contents(source, destination)
+ except OSError as e:
+ # Fast-copy failures name both files and can come from either
side, so only
+ # an error naming the source alone is attributed to it.
+ if e.filename == os.fspath(source) and e.filename2 is None:
+ raise_source_access_error(
+ e, subject=f"file {file_info['path']!r}",
phase="materializing the snapshot"
+ )
+ raise
+ # Executable bit is part of the content address; verify before
commit.
+ destination_sha256, destination_size =
compute_file_sha256(destination)
+ destination_executable =
bool(stat.S_IMODE(destination.stat().st_mode) & 0o111)
+ if (
+ destination_sha256 != file_info["sha256"]
+ or destination_size != file_info["size"]
+ or destination_executable != file_info["executable"]
+ ):
+ raise BundleManifestSourceChangedError(
+ f"Bundle source changed while materializing bundle version
{version}"
+ )
+
+ _write_manifest_file(manifest, tmp_path / MANIFEST_FILE_NAME)
+ _set_snapshot_permissions(tmp_path)
+ fsync_tree_directories(tmp_path)
+ try:
+ os.replace(tmp_path, version_path)
+ except OSError:
+ # Renaming onto another publisher's committed tree fails with a
+ # platform-specific errno, and NFS can report failure after the
rename
+ # completed, so check for the result instead.
+ if not version_path.is_dir():
+ raise
+ else:
+ return True
+ except Exception:
+ # Best effort only: a cleanup failure must not mask the original error.
+ with suppress(OSError):
+ remove_bundle_tree_forcefully(tmp_path)
+ raise
+ with suppress(OSError):
+ remove_bundle_tree_forcefully(tmp_path)
+ return False
+
+
+def _remove_orphaned_atomic_temp_files(destination_path: Path) -> None:
+ if not destination_path.parent.exists():
+ return
+ prefix = f".{destination_path.name}."
+ for path in destination_path.parent.iterdir():
+ if not path.name.startswith(prefix) or path.is_symlink() or not
path.is_file():
+ continue
+ with suppress(OSError):
+ path.unlink()
+
+
+def _has_content(path: Path, content: bytes) -> bool:
+ try:
+ return path.read_bytes() == content
+ except OSError:
+ return False
+
+
+def _write_json_atomically(destination_path: Path, payload: dict[str, Any]) ->
None:
+ content = serialize_bundle_version_manifest(payload)
+ fd, tmp_name = tempfile.mkstemp(
+ prefix=f".{destination_path.name}.",
+ dir=destination_path.parent,
+ )
+ tmp_path = Path(tmp_name)
+ version = payload.get("version")
+ ref_version = version if isinstance(version, str) else None
+ try:
+ with os.fdopen(fd, "wb") as file:
+ file.write(content)
+ file.flush()
+ # mkstemp creates the file owner-only; consumers can run as a
different OS user.
+ os.fchmod(file.fileno(), 0o644)
+ os.fsync(file.fileno())
+ try:
+ os.replace(tmp_path, destination_path)
+ except OSError as replace_error:
+ # NFS can report failure after completing the rename. The
temporary file still
+ # existing proves the reference was not replaced, but its absence
does not prove
+ # it was: a publisher whose lock is not enforced sweeps it as a
leftover.
+ try:
+ tmp_path.lstat()
+ except FileNotFoundError:
+ if not _has_content(destination_path, content):
+ raise BundleManifestRefDurabilityError(
+ ref_locator=str(destination_path), version=ref_version
+ ) from replace_error
+ except OSError as e:
+ raise BundleManifestRefDurabilityError(
+ ref_locator=str(destination_path), version=ref_version
+ ) from e
+ else:
+ raise replace_error
+ try:
+ fsync_directory(destination_path.parent)
+ except OSError as e:
+ raise BundleManifestRefDurabilityError(
+ ref_locator=str(destination_path), version=ref_version
+ ) from e
+ except Exception:
+ with suppress(OSError):
+ tmp_path.unlink(missing_ok=True)
+ raise
+
+
+def _ensure_public_dir(path: Path, *, root: Path, sync_existing: bool) -> None:
+ """
+ Create ``path`` and the missing directories between it and ``root``, which
must exist.
+
+ Each created directory is world-traversable and uses the group write and
setgid bits of
+ the provisioned ``root``, and is synced together with its parent. A
directory whose setup
+ fails is removed again: later calls skip existing directories, so they
would never finish
+ it. Pre-existing directories keep their permissions and remain the
deployer's contract.
+
+ With ``sync_existing``, the existing directories from ``path`` up to
``root`` are synced
+ too: a publisher killed after creating a directory, before syncing its
parent, leaves one
+ that later calls would otherwise never sync.
+ """
+ root_mode = get_path_mode(root)
+ if root_mode is None or not stat.S_ISDIR(root_mode):
+ raise BundleManifestError(
+ f"published_root {root} does not exist or is not a directory.
Create it before "
+ "publishing, and if it is on a shared filesystem, check that the
filesystem is mounted."
+ )
+ # Take the group bits from root, not from each new directory's parent: a
concurrent
+ # publisher may have created that parent and not yet set its final mode.
+ directory_mode = 0o755 | (stat.S_IMODE(root_mode) & (stat.S_ISGID |
stat.S_IWGRP))
+ missing: list[Path] = []
+ current = path
+ while current != root and get_path_mode(current) is None:
+ missing.append(current)
+ current = current.parent
+ if sync_existing:
+ for existing in (current, *current.parents):
+ fsync_directory(existing)
+ if existing == root:
+ break
+ for created in reversed(missing):
+ try:
+ created.mkdir()
+ except FileExistsError:
+ mode = get_path_mode(created)
+ if mode is None or not stat.S_ISDIR(mode):
+ raise
+ created_here = False
+ else:
+ created_here = True
+ try:
+ _apply_directory_mode(created, directory_mode,
created_here=created_here)
+ fsync_directory(created)
+ fsync_directory(created.parent)
+ except OSError:
+ if created_here:
+ _remove_unfinished_directory(created)
+ raise
+
+
+def _apply_directory_mode(path: Path, mode: int, *, created_here: bool) ->
None:
+ try:
+ path.chmod(mode)
+ except OSError:
+ # Another publisher may own the directory and apply its permissions
itself.
+ if created_here or path.stat().st_uid == os.geteuid():
+ raise
+
+
+def _remove_unfinished_directory(path: Path) -> None:
+ try:
+ path.rmdir()
+ fsync_directory(path.parent)
+ except OSError:
+ log.warning(
+ "Could not remove publication directory %s after its setup failed;
inspect it before retrying",
+ path,
+ exc_info=True,
+ )
+
+
+def _write_manifest_file(manifest: dict[str, Any], manifest_file: Path) ->
None:
+ manifest_file.write_bytes(serialize_bundle_version_manifest(manifest))
+
+
+def _set_snapshot_permissions(snapshot_path: Path) -> None:
+ # Snapshots must be read-only and world-readable: the publisher and the
Airflow
+ # components reading published_root may run as different OS users.
+ directories: list[Path] = []
+ for dirpath, _, filenames in os.walk(snapshot_path,
onerror=raise_walk_error):
+ directories.append(Path(dirpath))
+ for filename in filenames:
+ path = Path(dirpath) / filename
+ if path.is_symlink():
+ continue
+ executable = stat.S_IMODE(path.stat().st_mode) & 0o111
+ path.chmod(0o555 if executable else 0o444)
+ # One sync after chmod makes the data and the final mode durable
together.
+ fsync_file(path)
+ for directory in sorted(directories, key=lambda path: len(path.parts),
reverse=True):
+ directory.chmod(0o555)
diff --git a/airflow-core/src/airflow/dag_processing/bundles/manifest.py
b/airflow-core/src/airflow/dag_processing/bundles/manifest.py
index 9a3e141f030..91a1ba46c6a 100644
--- a/airflow-core/src/airflow/dag_processing/bundles/manifest.py
+++ b/airflow-core/src/airflow/dag_processing/bundles/manifest.py
@@ -98,7 +98,7 @@ class BundleVersionManifest:
source_snapshot: BundleSourceSnapshot
-def _raise_source_access_error(error: OSError, *, subject: str, phase: str) ->
NoReturn:
+def raise_source_access_error(error: OSError, *, subject: str, phase: str) ->
NoReturn:
"""
Re-raise an OSError from the bundle source as the matching manifest error.
@@ -106,6 +106,8 @@ def _raise_source_access_error(error: OSError, *, subject:
str, phase: str) -> N
treats as "nothing stable to publish yet, try the next pass". Any other
OSError --
a permission problem, a stale network handle -- is a standing fault, so it
must not
be reported as a benign concurrent edit and then quietly retried forever.
+
+ :meta private:
"""
if isinstance(error, FileNotFoundError):
raise BundleManifestSourceChangedError(
@@ -115,7 +117,7 @@ def _raise_source_access_error(error: OSError, *, subject:
str, phase: str) -> N
def _raise_source_walk_error(error: OSError) -> NoReturn:
- _raise_source_access_error(
+ raise_source_access_error(
error, subject=f"directory {error.filename}", phase="collecting
manifest metadata"
)
@@ -134,7 +136,7 @@ def _iter_manifest_file_paths(root: Path) ->
Iterator[tuple[Path, os.stat_result
try:
file_stat = path.lstat()
except OSError as e:
- _raise_source_access_error(
+ raise_source_access_error(
e, subject=f"directory {path}", phase="collecting manifest
metadata"
)
if stat.S_ISLNK(file_stat.st_mode):
@@ -151,7 +153,7 @@ def _iter_manifest_file_paths(root: Path) ->
Iterator[tuple[Path, os.stat_result
try:
file_stat = path.lstat()
except OSError as e:
- _raise_source_access_error(e, subject=f"file {path}",
phase="collecting manifest metadata")
+ raise_source_access_error(e, subject=f"file {path}",
phase="collecting manifest metadata")
if stat.S_ISLNK(file_stat.st_mode):
raise BundleManifestError(f"Bundle source contains symlinked
file: {path}")
if not stat.S_ISREG(file_stat.st_mode):
@@ -301,7 +303,7 @@ def _ensure_source_file_unchanged(source_file:
BundleSourceFile) -> None:
try:
current_stat = source_file.path.lstat()
except OSError as e:
- _raise_source_access_error(
+ raise_source_access_error(
e, subject=f"file {source_file.relative_path}", phase="verifying
manifest metadata"
)
@@ -398,7 +400,7 @@ def build_bundle_version_manifest(
try:
file_digest, _ = compute_file_sha256(source_file.path)
except OSError as e:
- _raise_source_access_error(
+ raise_source_access_error(
e, subject=f"file {source_file.relative_path}",
phase="building manifest"
)
else:
diff --git a/airflow-core/tests/unit/dag_processing/bundles/test__filesystem.py
b/airflow-core/tests/unit/dag_processing/bundles/test__filesystem.py
new file mode 100644
index 00000000000..b632fb83a35
--- /dev/null
+++ b/airflow-core/tests/unit/dag_processing/bundles/test__filesystem.py
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import errno
+import os
+from pathlib import Path
+from unittest import mock
+
+import pytest
+
+from airflow.dag_processing.bundles import _filesystem as filesystem_module
+from airflow.dag_processing.bundles._filesystem import (
+ fsync_directory,
+ fsync_tree_directories,
+ read_json_file,
+ remove_bundle_tree_forcefully,
+)
+
+
+def test_read_json_file_decodes_utf8_regardless_of_locale(tmp_path):
+ path = tmp_path / "payload.json"
+ path.write_bytes('{"value": "valid UTF-8: ☃"}'.encode())
+
+ with mock.patch.object(Path, "read_text", autospec=True,
side_effect=Path.read_text) as mock_read:
+ payload = read_json_file(path, missing_message="missing",
invalid_message="invalid")
+
+ assert payload == {"value": "valid UTF-8: ☃"}
+ mock_read.assert_called_once_with(path, encoding="utf-8")
+
+
[email protected](
+ filesystem_module.os, "open", autospec=True,
side_effect=PermissionError("cannot open directory")
+)
+def test_fsync_directory_propagates_open_failure(mock_open, tmp_path):
+ with pytest.raises(PermissionError, match="cannot open directory"):
+ fsync_directory(tmp_path)
+
+
+_real_scandir = os.scandir
+
+
+def _scandir_unless_unlistable(path="."):
+ """List directories, failing for any named ``unlistable`` as an I/O fault
would."""
+ if os.path.basename(os.fspath(path)) == "unlistable":
+ raise PermissionError(errno.EACCES, "cannot list directory",
os.fspath(path))
+ return _real_scandir(path)
+
+
[email protected](filesystem_module.os, "scandir", autospec=True,
side_effect=_scandir_unless_unlistable)
+def test_fsync_tree_directories_fails_on_an_unlistable_directory(mock_scandir,
tmp_path):
+ (tmp_path / "tree" / "unlistable").mkdir(parents=True)
+
+ with pytest.raises(PermissionError, match="cannot list directory"):
+ fsync_tree_directories(tmp_path / "tree")
+
+
+class TestRemoveBundleTreeForcefully:
+ def test_removes_read_only_tree(self, tmp_path):
+ tree = tmp_path / "tree"
+ (tree / "dags").mkdir(parents=True)
+ (tree / "dags" / "example.py").write_text("print('dag')")
+ (tree / "dags" / "example.py").chmod(0o444)
+ (tree / "dags").chmod(0o555)
+ tree.chmod(0o555)
+
+ remove_bundle_tree_forcefully(tree)
+
+ assert not tree.exists()
+
+ def test_unlinks_a_symlink_without_following_it(self, tmp_path):
+ target = tmp_path / "target"
+ target.mkdir()
+ link = tmp_path / "link"
+ link.symlink_to(target, target_is_directory=True)
+
+ remove_bundle_tree_forcefully(link)
+
+ assert not link.exists()
+ assert target.is_dir()
diff --git
a/airflow-core/tests/unit/dag_processing/bundles/test_filesystem_artifact_store.py
b/airflow-core/tests/unit/dag_processing/bundles/test_filesystem_artifact_store.py
new file mode 100644
index 00000000000..d6e5b3d742c
--- /dev/null
+++
b/airflow-core/tests/unit/dag_processing/bundles/test_filesystem_artifact_store.py
@@ -0,0 +1,1407 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Tests for the shared-filesystem artifact store and its filesystem
helpers."""
+
+from __future__ import annotations
+
+import errno
+import json
+import multiprocessing
+import os
+import shutil
+import stat
+from fcntl import F_GETFL, LOCK_EX, LOCK_NB, LOCK_UN, fcntl, flock
+from functools import partial
+from pathlib import Path
+from typing import TYPE_CHECKING
+from unittest import mock
+
+import pytest
+
+from airflow.dag_processing.bundles import (
+ _filesystem as filesystem_module,
+ filesystem_artifact_store as store_module,
+)
+from airflow.dag_processing.bundles._filesystem import read_json_file
+from airflow.dag_processing.bundles.artifact_store import (
+ ArtifactStore,
+ BundleManifestNotFoundError,
+ BundleManifestRefDurabilityError,
+ PublishedSnapshot,
+)
+from airflow.dag_processing.bundles.filesystem_artifact_store import
FilesystemArtifactStore
+from airflow.dag_processing.bundles.manifest import (
+ MANIFEST_FILE_NAME,
+ BundleManifestError,
+ BundleManifestSourceChangedError,
+ build_bundle_version_manifest,
+)
+
+if TYPE_CHECKING:
+ from multiprocessing.connection import Connection
+
+BUNDLE_NAME = "contract"
+
+
+def _create_published_root(path: Path) -> Path:
+ path.mkdir(parents=True)
+ # Created directories copy the root's group bits, so pin its mode against
the umask.
+ path.chmod(0o755)
+ return path
+
+
[email protected]
+def store(tmp_path):
+ return FilesystemArtifactStore(
+ bundle_name=BUNDLE_NAME,
+ published_root=_create_published_root(tmp_path / "published"),
+ )
+
+
+def _validate_tree(tree: Path) -> None:
+ """Spec for snapshot validator mocks, so a wrong callback signature fails
the test."""
+
+
+_real_replace = os.replace
+
+
+def _replace_then_fail(src, dst):
+ """Complete the rename, then report failure the way a lost NFS reply
does."""
+ _real_replace(src, dst)
+ raise OSError(errno.EIO, "rename reply lost")
+
+
+def _sweep_then_fail(src, dst):
+ """Lose the temporary file to another publisher's leftover sweep before
the rename."""
+ os.unlink(src)
+ raise FileNotFoundError(errno.ENOENT, "No such file or directory",
os.fspath(src))
+
+
+_real_scandir = os.scandir
+
+
+def _scandir_unless_unlistable(path="."):
+ """List directories, failing for any named ``unlistable`` as an I/O fault
would."""
+ if os.path.basename(os.fspath(path)) == "unlistable":
+ raise PermissionError(errno.EACCES, "cannot list directory",
os.fspath(path))
+ return _real_scandir(path)
+
+
+def _fail_fast_copy(src, dst, **kwargs):
+ """Fail the way shutil's fast-copy paths do: the error names both files."""
+ raise OSError(errno.ENOSPC, "No space left on device", os.fspath(src),
None, os.fspath(dst))
+
+
+def _is_dir_as_in_python_314(path, *, follow_symlinks=True):
+ """``Path.is_dir()`` as of Python 3.14, which returns False on any
OSError."""
+ return os.path.isdir(path)
+
+
+def _exists_as_in_python_314(path, *, follow_symlinks=True):
+ """``Path.exists()`` as of Python 3.14, which returns False on any
OSError."""
+ return os.path.exists(path)
+
+
+def _fail_unlock(lock_file, operation):
+ if operation == LOCK_UN:
+ raise OSError(errno.EIO, "unlock failed")
+ return flock(lock_file, operation)
+
+
+def _get_lock_path(store: FilesystemArtifactStore) -> Path:
+ return store.published_root / "_locks" / f"{BUNDLE_NAME}.lock"
+
+
+def _get_ref_path(store: FilesystemArtifactStore) -> Path:
+ return store.published_root / "refs" / BUNDLE_NAME / "latest.json"
+
+
+def _get_snapshots_dir(store: FilesystemArtifactStore) -> Path:
+ return store.published_root / "versions" / BUNDLE_NAME
+
+
+def _write_source(root: Path) -> Path:
+ (root / "dags").mkdir(parents=True, exist_ok=True)
+ (root / "dags" / "example.py").write_text("print('dag')\n")
+ return root
+
+
+def _publish(store, source: Path):
+ result = build_bundle_version_manifest(bundle_name=BUNDLE_NAME,
root=source, backend_type="local")
+ store.prepare_publish_areas()
+ publication = store.publish_snapshot(
+ manifest=result.manifest,
+ source_root=source,
+ validate_existing=lambda tree: None,
+ )
+ return result, publication
+
+
+def _acquire_guard_after_contention(store: FilesystemArtifactStore, status:
Connection) -> None:
+ reported = False
+
+ def acquire_lock(lock_file, operation):
+ nonlocal reported
+ try:
+ flock(lock_file, operation)
+ except BlockingIOError:
+ if not reported:
+ status.send("blocked")
+ reported = True
+ raise
+ if operation & LOCK_EX and not reported:
+ status.send("unlocked")
+ reported = True
+
+ with status, mock.patch.object(store_module, "flock", autospec=True,
side_effect=acquire_lock):
+ with store.publication_guard():
+ status.send("acquired")
+
+
+def test_not_found_error_is_also_a_file_not_found_error():
+ error = BundleManifestNotFoundError("missing")
+ assert isinstance(error, BundleManifestError)
+ assert isinstance(error, FileNotFoundError)
+
+
+def test_store_without_backend_type_cannot_be_instantiated():
+ methods = {
+ name: lambda *args, **kwargs: None for name in
ArtifactStore.__abstractmethods__ - {"backend_type"}
+ }
+ store_class = type("StoreWithoutBackendType", (ArtifactStore,), methods)
+
+ with pytest.raises(TypeError, match="backend_type"):
+ store_class()
+
+
+class TestFilesystemDocuments:
+ def test_read_ref_missing_raises_not_found(self, store):
+ with pytest.raises(
+ BundleManifestNotFoundError, match=f"Bundle '{BUNDLE_NAME}' has no
release reference"
+ ):
+ store.read_ref()
+
+ def test_ref_roundtrip(self, store):
+ store.prepare_publish_areas()
+ store.write_ref({"schema_version": 1, "bundle_name": BUNDLE_NAME})
+ payload = store.read_ref()
+ assert payload == {"schema_version": 1, "bundle_name": BUNDLE_NAME}
+
+ @pytest.mark.parametrize("content", [b"{not json", b"\xff", b"[1, 2]"])
+ def test_read_ref_invalid_document_raises(self, store, content):
+ store.prepare_publish_areas()
+ _get_ref_path(store).write_bytes(content)
+ with pytest.raises(BundleManifestError, match="release reference .* is
not a JSON object"):
+ store.read_ref()
+
+
+class TestFilesystemSnapshots:
+ def test_publish_then_exists_then_fetch(self, store, tmp_path):
+ source = _write_source(tmp_path / "source")
+ script = source / "bin" / "task.sh"
+ script.parent.mkdir()
+ script.write_text("#!/bin/sh\n")
+ script.chmod(0o755)
+ result, publication = _publish(store, source)
+
+ assert publication == PublishedSnapshot(manifest=result.manifest,
created=True)
+ assert store.snapshot_exists(result.version)
+
+ seen_trees: list[Path] = []
+ destination = tmp_path / "dest"
+ destination.mkdir()
+ store.fetch_snapshot(result.version, destination,
structural_validator=seen_trees.append)
+ assert seen_trees == [_get_snapshots_dir(store) / result.version]
+ assert (destination / "dags" / "example.py").read_text() ==
"print('dag')\n"
+ assert stat.S_IMODE((destination / "dags" /
"example.py").stat().st_mode) == 0o444
+ assert stat.S_IMODE((destination / "bin" / "task.sh").stat().st_mode)
== 0o555
+ for directory in (destination / "dags", destination / "bin"):
+ assert stat.S_IMODE(directory.stat().st_mode) & stat.S_IWUSR
+ embedded = json.loads((destination / MANIFEST_FILE_NAME).read_text())
+ assert embedded["version"] == result.version
+
+ def test_fetch_rejects_non_empty_destination(self, store, tmp_path):
+ result, _ = _publish(store, _write_source(tmp_path / "source"))
+ destination = tmp_path / "dest"
+ store.fetch_snapshot(result.version, destination,
structural_validator=lambda tree: None)
+ structural_validator = mock.create_autospec(_validate_tree)
+
+ with pytest.raises(ValueError, match="must be absent or empty"):
+ store.fetch_snapshot(result.version, destination,
structural_validator=structural_validator)
+
+ structural_validator.assert_not_called()
+
+ def test_fetch_validates_before_copying(self, store, tmp_path):
+ result, _ = _publish(store, _write_source(tmp_path / "source"))
+ destination = tmp_path / "dest"
+ structural_validator = mock.create_autospec(
+ _validate_tree, side_effect=BundleManifestError("snapshot
rejected")
+ )
+
+ with pytest.raises(BundleManifestError, match="snapshot rejected"):
+ store.fetch_snapshot(result.version, destination,
structural_validator=structural_validator)
+
+ structural_validator.assert_called_once_with(_get_snapshots_dir(store)
/ result.version)
+ assert not destination.exists()
+
+ def test_publish_is_idempotent_and_validates_existing(self, store,
tmp_path):
+ source = _write_source(tmp_path / "source")
+ result, publication = _publish(store, source)
+ assert publication.created is True
+
+ validated: list[Path] = []
+ republication = store.publish_snapshot(
+ manifest=result.manifest,
+ source_root=source,
+ validate_existing=validated.append,
+ )
+ assert republication == PublishedSnapshot(manifest=result.manifest,
created=False)
+ assert validated == [_get_snapshots_dir(store) / result.version]
+
+ def
test_republish_keeps_the_existing_snapshot_and_returns_its_manifest(self,
store, tmp_path):
+ source = _write_source(tmp_path / "source")
+ first, _ = _publish(store, source)
+ # Object version ids are bookkeeping outside the version: same
version, different bytes.
+ republished = build_bundle_version_manifest(
+ bundle_name=BUNDLE_NAME,
+ root=source,
+ backend_type="local",
+ file_version_ids={"dags/example.py": "object-version-2"},
+ )
+ assert republished.version == first.version
+ manifest_file = _get_snapshots_dir(store) / first.version /
MANIFEST_FILE_NAME
+ committed_bytes = manifest_file.read_bytes()
+ validate_existing = mock.create_autospec(_validate_tree)
+
+ publication = store.publish_snapshot(
+ manifest=republished.manifest, source_root=source,
validate_existing=validate_existing
+ )
+
+ assert publication == PublishedSnapshot(manifest=first.manifest,
created=False)
+ validate_existing.assert_called_once_with(_get_snapshots_dir(store) /
first.version)
+ assert manifest_file.read_bytes() == committed_bytes
+
+ def test_fetch_unpublished_version_raises_not_found(self, store, tmp_path):
+ destination = tmp_path / "dest"
+ destination.mkdir()
+ with pytest.raises(BundleManifestNotFoundError, match="is not
published"):
+ store.fetch_snapshot(
+ "sha256-" + "0" * 64,
+ destination,
+ structural_validator=lambda tree: None,
+ )
+
+ def test_snapshot_exists_is_false_before_publication(self, store):
+ assert store.snapshot_exists("sha256-" + "0" * 64) is False
+
+ def test_publication_guard_removes_orphans(self, store):
+ store.prepare_publish_areas()
+ orphan_snapshot = _get_snapshots_dir(store) / (".sha256-" + "0" * 64 +
".tmp123")
+ orphan_snapshot.mkdir(parents=True)
+ orphan_ref = _get_ref_path(store).parent /
f".{_get_ref_path(store).name}.tmp123"
+ orphan_ref.write_text("partial")
+
+ with store.publication_guard():
+ pass
+
+ assert not orphan_snapshot.exists()
+ assert not orphan_ref.exists()
+
+ @mock.patch.object(store_module, "remove_orphaned_temp_snapshots",
autospec=True)
+ @mock.patch.object(store_module, "flock", autospec=True)
+ def test_publication_guard_sweeps_only_while_holding_the_lock(self,
mock_flock, mock_sweep, store):
+ calls: list[str] = []
+ mock_flock.side_effect = lambda lock_file, operation:
calls.append(f"flock-{operation}")
+ mock_sweep.side_effect = lambda versions_dir: calls.append("sweep")
+
+ with store.publication_guard():
+ calls.append("body")
+
+ assert calls == [f"flock-{LOCK_EX | LOCK_NB}", "sweep", "body",
f"flock-{LOCK_UN}"]
+
+ @pytest.mark.parametrize(
+ "name",
+ [
+ ".sha256-" + "0" * 64,
+ ".sha256-" + "0" * 64 + ".",
+ ".sha256-" + "A" * 64 + ".tmp123",
+ ".sha256-" + "0" * 63 + ".tmp123",
+ "sha256-" + "0" * 64,
+ ],
+ )
+ def test_publication_guard_keeps_non_temp_directories(self, store, name):
+ store.prepare_publish_areas()
+ directory = _get_snapshots_dir(store) / name
+ directory.mkdir()
+
+ with store.publication_guard():
+ pass
+
+ assert directory.exists()
+
+
+class TestFilesystemCoordination:
+ def test_publication_guard_serializes_processes(self, store):
+ context = multiprocessing.get_context("spawn")
+ receiver, sender = context.Pipe(duplex=False)
+ process = context.Process(target=_acquire_guard_after_contention,
args=(store, sender))
+
+ try:
+ with store.publication_guard():
+ process.start()
+ sender.close()
+ assert receiver.poll(30), "second publisher did not test the
held lock"
+ assert receiver.recv() == "blocked"
+
+ assert receiver.poll(30), "second publisher did not acquire the
released lock"
+ assert receiver.recv() == "acquired"
+ process.join(30)
+ assert process.exitcode == 0
+ finally:
+ receiver.close()
+ sender.close()
+ if process.pid is not None:
+ if process.is_alive():
+ process.terminate()
+ process.join(30)
+
+ def test_locators_are_opaque_strings_scoped_to_the_bundle(self, store,
tmp_path):
+ version = "sha256-" + "a" * 64
+ published_root = tmp_path / "published"
+ assert store.published_root == published_root
+ assert store.ref_locator == str(published_root / "refs" / BUNDLE_NAME
/ "latest.json")
+ assert store.snapshot_locator(version) == str(published_root /
"versions" / BUNDLE_NAME / version)
+
+
+class TestFilesystemPublication:
+ """Durability and permission behavior specific to the shared-filesystem
layout."""
+
+ def
test_publication_creates_readable_artifacts_and_group_writable_lock(self,
store, tmp_path):
+ source = _write_source(tmp_path / "source")
+ script = source / "bin" / "task.sh"
+ script.parent.mkdir()
+ script.write_text("#!/bin/sh\n")
+ script.chmod(0o700)
+
+ old_umask = os.umask(0o077)
+ try:
+ with store.publication_guard():
+ result, _ = _publish(store, source)
+ store.write_ref(result.ref_payload)
+ finally:
+ os.umask(old_umask)
+
+ def get_mode(path: Path) -> int:
+ return stat.S_IMODE(path.stat().st_mode)
+
+ snapshot = _get_snapshots_dir(store) / result.version
+ assert get_mode(_get_ref_path(store)) == 0o644
+ assert get_mode(_get_lock_path(store)) == 0o664
+ assert get_mode(_get_ref_path(store).parent) == 0o755
+ assert get_mode(_get_snapshots_dir(store)) == 0o755
+ assert get_mode(snapshot) == 0o555
+ assert get_mode(snapshot / "dags") == 0o555
+ assert get_mode(snapshot / "dags" / "example.py") == 0o444
+ assert get_mode(snapshot / "bin" / "task.sh") == 0o555
+ assert get_mode(snapshot / MANIFEST_FILE_NAME) == 0o444
+
+ def test_prepare_publish_areas_keeps_pre_existing_root_mode(self, store):
+ store.published_root.chmod(0o750)
+
+ store.prepare_publish_areas()
+
+ assert stat.S_IMODE(store.published_root.stat().st_mode) == 0o750
+
+ @pytest.mark.parametrize("root_state", ["missing", "file"])
+ @pytest.mark.parametrize("operation", ["prepare_publish_areas",
"publication_guard", "write_ref"])
+ def test_publication_requires_an_existing_published_root(self, tmp_path,
operation, root_state):
+ mount_point = tmp_path / "share"
+ mount_point.mkdir()
+ published_root = mount_point / "published"
+ if root_state == "file":
+ published_root.write_text("not a directory")
+ store = FilesystemArtifactStore(bundle_name=BUNDLE_NAME,
published_root=published_root)
+ operations = {
+ "prepare_publish_areas": store.prepare_publish_areas,
+ "publication_guard": lambda: store.publication_guard().__enter__(),
+ "write_ref": partial(store.write_ref, {"version": "v1"}),
+ }
+
+ with pytest.raises(BundleManifestError, match="does not exist or is
not a directory") as raised:
+ operations[operation]()
+
+ assert type(raised.value) is BundleManifestError
+ assert list(mount_point.iterdir()) == ([published_root] if root_state
== "file" else [])
+
+ def
test_created_directories_inherit_the_group_permissions_of_published_root(self,
store, tmp_path):
+ store.published_root.chmod(0o2775)
+ source = _write_source(tmp_path / "source")
+
+ old_umask = os.umask(0o077)
+ try:
+ with store.publication_guard():
+ result, _ = _publish(store, source)
+ store.write_ref(result.ref_payload)
+ finally:
+ os.umask(old_umask)
+
+ created = [
+ _get_lock_path(store).parent,
+ _get_snapshots_dir(store).parent,
+ _get_snapshots_dir(store),
+ _get_ref_path(store).parent.parent,
+ _get_ref_path(store).parent,
+ ]
+ assert {path: stat.S_IMODE(path.stat().st_mode) for path in created}
== dict.fromkeys(created, 0o2775)
+
+ @pytest.mark.parametrize("area", ["versions", "refs"])
+ @mock.patch.object(Path, "mkdir", autospec=True, side_effect=Path.mkdir)
+ def
test_concurrent_bundles_keep_group_write_during_directory_creation(self,
mock_mkdir, store, area):
+ published_root = store.published_root
+ published_root.chmod(0o2775)
+ shared_directory = published_root / area
+ rival = FilesystemArtifactStore(bundle_name="rival",
published_root=published_root)
+ real_mkdir = mock_mkdir.side_effect
+
+ def prepare_rival_before_chmod(path, *args, **kwargs):
+ result = real_mkdir(path, *args, **kwargs)
+ if path == shared_directory:
+ rival.prepare_publish_areas()
+ return result
+
+ mock_mkdir.side_effect = prepare_rival_before_chmod
+ previous_umask = os.umask(0o077)
+ try:
+ store.prepare_publish_areas()
+ finally:
+ os.umask(previous_umask)
+
+ directories = [
+ shared_directory,
+ shared_directory / store.bundle_name,
+ shared_directory / rival.bundle_name,
+ ]
+ assert {path: stat.S_IMODE(path.stat().st_mode) for path in
directories} == dict.fromkeys(
+ directories, 0o2775
+ )
+
+ @pytest.mark.parametrize(("original_mode", "expected_mode"), [(0o600,
0o664), (0o666, 0o666)])
+ def
test_publication_guard_grants_group_access_without_removing_permissions(
+ self, store, original_mode, expected_mode
+ ):
+ _get_lock_path(store).parent.mkdir(parents=True)
+ _get_lock_path(store).touch()
+ _get_lock_path(store).chmod(original_mode)
+
+ with store.publication_guard():
+ pass
+
+ assert stat.S_IMODE(_get_lock_path(store).stat().st_mode) ==
expected_mode
+
+ @mock.patch.object(store_module, "flock", autospec=True)
+ def test_publication_guard_locks_a_writable_descriptor(self, mock_flock,
store):
+ def check_descriptor(lock_file, operation):
+ if operation & LOCK_EX:
+ assert fcntl(lock_file, F_GETFL) & os.O_ACCMODE == os.O_RDWR
+ assert lock_file.readable()
+ assert lock_file.writable()
+ flock(lock_file, operation)
+
+ mock_flock.side_effect = check_descriptor
+
+ with store.publication_guard():
+ pass
+
+ assert mock_flock.call_count == 2
+
+ def test_publication_guard_explains_inaccessible_lock_file(self, store):
+ real_open = os.open
+
+ def deny_lock_open(path, *args, **kwargs):
+ if Path(path) == _get_lock_path(store):
+ raise PermissionError("denied")
+ return real_open(path, *args, **kwargs)
+
+ with (
+ mock.patch.object(store_module.os, "open", autospec=True,
side_effect=deny_lock_open),
+ pytest.raises(BundleManifestError, match="read/write"),
+ store.publication_guard(),
+ ):
+ pass
+
+ @mock.patch.object(store_module, "time", autospec=True)
+ @mock.patch.object(store_module, "flock", autospec=True)
+ def test_publication_guard_waits_for_a_held_lock(self, mock_flock,
mock_time, store):
+ mock_time.monotonic.return_value = 0.0
+ mock_flock.side_effect = [BlockingIOError(), BlockingIOError(), None,
None]
+ entered = []
+
+ with store.publication_guard():
+ entered.append(True)
+
+ assert entered == [True]
+ assert mock_time.sleep.call_count == 2
+
+ @mock.patch.object(store_module, "log", autospec=True)
+ @mock.patch.object(store_module, "time", autospec=True)
+ @mock.patch.object(store_module, "flock", autospec=True,
side_effect=BlockingIOError())
+ def test_publication_guard_times_out_and_logs_while_waiting(
+ self, mock_flock, mock_time, mock_log, tmp_path
+ ):
+ clock = 0.0
+
+ def advance(seconds):
+ nonlocal clock
+ clock += seconds
+
+ mock_time.monotonic.side_effect = lambda: clock
+ mock_time.sleep.side_effect = advance
+ store = FilesystemArtifactStore(
+ bundle_name=BUNDLE_NAME,
+ published_root=_create_published_root(tmp_path / "published"),
+ lock_timeout=65,
+ )
+
+ with (
+ pytest.raises(BundleManifestError, match="Timed out after 65s
waiting for publication lock"),
+ store.publication_guard(),
+ ):
+ pass
+
+ assert clock == 65
+ assert (
+ mock_log.info.call_args_list
+ == [mock.call("Waiting for publication lock %s held by another
publisher", _get_lock_path(store))]
+ * 3
+ )
+
+ @mock.patch.object(store_module, "flock", autospec=True,
side_effect=OSError(errno.ENOLCK, "No locks"))
+ def test_publication_guard_reports_unsupported_locking(self, mock_flock,
store):
+ with (
+ pytest.raises(BundleManifestError, match="must support flock
across hosts") as raised,
+ store.publication_guard(),
+ ):
+ pass
+
+ assert raised.value.__cause__ is mock_flock.side_effect
+
+ @pytest.mark.parametrize("artifact", ["snapshot", "reference"])
+ def test_publication_fsyncs_created_hierarchy_before_commit(self, store,
tmp_path, artifact):
+ source = _write_source(tmp_path / "source")
+ calls: list[tuple[str, Path]] = []
+ real_replace = os.replace
+
+ def record_replace(source_path, destination_path):
+ calls.append(("replace", Path(destination_path)))
+ real_replace(source_path, destination_path)
+
+ with (
+ mock.patch.object(
+ store_module,
+ "fsync_directory",
+ autospec=True,
+ side_effect=lambda path: calls.append(("fsync", path)),
+ ),
+ mock.patch.object(store_module.os, "replace", autospec=True,
side_effect=record_replace),
+ ):
+ if artifact == "snapshot":
+ result, _ = _publish(store, source)
+ destination = _get_snapshots_dir(store) / result.version
+ else:
+ store.write_ref({"schema_version": 1})
+ destination = _get_ref_path(store)
+
+ commit_index = calls.index(("replace", destination))
+ directory = destination.parent
+ while True:
+ assert ("fsync", directory) in calls[:commit_index]
+ if directory == store.published_root:
+ break
+ directory = directory.parent
+ assert ("fsync", destination.parent) in calls[commit_index + 1 :]
+
+ @pytest.mark.parametrize("artifact", ["snapshot", "reference"])
+ def
test_publication_removes_a_directory_whose_sync_failed_and_retry_syncs_it(
+ self, store, tmp_path, artifact
+ ):
+ source = _write_source(tmp_path / "source")
+ # The first directory each publication creates below published_root.
+ interrupted_directory = (
+ _get_snapshots_dir(store).parent if artifact == "snapshot" else
_get_ref_path(store).parent.parent
+ )
+ calls: list[tuple[str, Path]] = []
+ real_replace = os.replace
+ fail_sync = True
+
+ def record_sync(path):
+ nonlocal fail_sync
+ calls.append(("fsync", path))
+ if fail_sync and path == interrupted_directory:
+ fail_sync = False
+ raise OSError("sync interrupted")
+
+ def record_replace(source_path, destination_path):
+ calls.append(("replace", Path(destination_path)))
+ real_replace(source_path, destination_path)
+
+ publish = (
+ partial(_publish, store, source)
+ if artifact == "snapshot"
+ else partial(store.write_ref, {"schema_version": 1})
+ )
+ with (
+ mock.patch.object(store_module, "fsync_directory", autospec=True,
side_effect=record_sync),
+ mock.patch.object(store_module.os, "replace", autospec=True,
side_effect=record_replace),
+ ):
+ with pytest.raises(BundleManifestError, match="sync interrupted"):
+ publish()
+ assert not interrupted_directory.exists()
+ calls.clear()
+
+ publish()
+
+ commit_index = next(index for index, (operation, _) in
enumerate(calls) if operation == "replace")
+ assert ("fsync", interrupted_directory) in calls[:commit_index]
+ assert ("fsync", interrupted_directory.parent) in calls[:commit_index]
+
+ @pytest.mark.parametrize("artifact", ["snapshot", "reference"])
+ @mock.patch.object(store_module.os, "replace", autospec=True)
+ @mock.patch.object(store_module, "fsync_directory", autospec=True,
side_effect=OSError("sync failed"))
+ def test_publication_stops_when_hierarchy_sync_fails(
+ self, mock_fsync, mock_replace, store, tmp_path, artifact
+ ):
+ source = _write_source(tmp_path / "source")
+ publish = (
+ partial(_publish, store, source)
+ if artifact == "snapshot"
+ else partial(store.write_ref, {"schema_version": 1})
+ )
+
+ with pytest.raises(BundleManifestError, match="sync failed"):
+ publish()
+
+ mock_replace.assert_not_called()
+
+ @mock.patch.object(
+ store_module, "fsync_tree_directories", autospec=True,
side_effect=RuntimeError("failed to sync")
+ )
+ def test_publish_snapshot_removes_read_only_temp_tree_on_failure(self,
mock_fsync, store, tmp_path):
+ source = _write_source(tmp_path / "source")
+
+ with pytest.raises(RuntimeError, match="failed to sync"):
+ _publish(store, source)
+
+ assert list(_get_snapshots_dir(store).iterdir()) == []
+
+ @pytest.mark.parametrize(
+ "drift",
+ [
+ pytest.param(lambda dst: os.chmod(dst, 0o755),
id="executable-bit"),
+ pytest.param(lambda dst: Path(dst).write_text("print('bad')\n"),
id="same-size-content"),
+ pytest.param(lambda dst:
Path(dst).write_text("print('changed')\n"), id="size"),
+ ],
+ )
+ def test_publish_snapshot_rejects_source_drift_before_commit(self, store,
tmp_path, drift):
+ # The exec bit is part of the content address; failing only after
os.replace would
+ # permanently poison the published snapshot for that content version.
+ source = _write_source(tmp_path / "source")
+ real_copy = shutil.copy
+
+ def drift_after_copy(src, dst, **kwargs):
+ copied = real_copy(src, dst, **kwargs)
+ drift(dst)
+ return copied
+
+ with (
+ mock.patch.object(store_module.shutil, "copy", autospec=True,
side_effect=drift_after_copy),
+ pytest.raises(BundleManifestSourceChangedError, match="source
changed"),
+ ):
+ _publish(store, source)
+
+ assert list(_get_snapshots_dir(store).iterdir()) == []
+
+ def test_publish_snapshot_validates_a_version_committed_concurrently(self,
store, tmp_path):
+ source = _write_source(tmp_path / "source")
+ result = build_bundle_version_manifest(bundle_name=BUNDLE_NAME,
root=source, backend_type="local")
+ store.prepare_publish_areas()
+ rival = FilesystemArtifactStore(bundle_name=BUNDLE_NAME,
published_root=tmp_path / "published")
+ real_fsync_tree_directories = store_module.fsync_tree_directories
+ raced = False
+
+ def commit_rival_first(path):
+ nonlocal raced
+ if not raced:
+ raced = True
+ rival.publish_snapshot(
+ manifest=result.manifest, source_root=source,
validate_existing=lambda tree: None
+ )
+ real_fsync_tree_directories(path)
+
+ validate_existing = mock.create_autospec(_validate_tree)
+ with mock.patch.object(
+ store_module, "fsync_tree_directories", autospec=True,
side_effect=commit_rival_first
+ ):
+ publication = store.publish_snapshot(
+ manifest=result.manifest, source_root=source,
validate_existing=validate_existing
+ )
+
+ assert publication.created is False
+ validate_existing.assert_called_once_with(_get_snapshots_dir(store) /
result.version)
+ assert list(_get_snapshots_dir(store).iterdir()) ==
[_get_snapshots_dir(store) / result.version]
+
+ @mock.patch.object(store_module.os, "replace", autospec=True,
side_effect=_replace_then_fail)
+ @mock.patch.object(
+ store_module, "fsync_directory", autospec=True,
side_effect=store_module.fsync_directory
+ )
+ def test_publish_snapshot_syncs_a_rename_that_reported_failure(
+ self, mock_fsync, mock_replace, store, tmp_path
+ ):
+ source = _write_source(tmp_path / "source")
+ result = build_bundle_version_manifest(bundle_name=BUNDLE_NAME,
root=source, backend_type="local")
+ store.prepare_publish_areas()
+ mock_fsync.reset_mock()
+ validate_existing = mock.create_autospec(_validate_tree)
+
+ publication = store.publish_snapshot(
+ manifest=result.manifest, source_root=source,
validate_existing=validate_existing
+ )
+
+ assert publication.created is False
+ mock_replace.assert_called_once()
+ mock_fsync.assert_called_once_with(_get_snapshots_dir(store))
+ validate_existing.assert_called_once_with(_get_snapshots_dir(store) /
result.version)
+ assert list(_get_snapshots_dir(store).iterdir()) ==
[_get_snapshots_dir(store) / result.version]
+
+ @mock.patch.object(store_module.os, "replace", autospec=True,
side_effect=OSError(errno.EIO, "I/O error"))
+ def test_publish_snapshot_reraises_a_failed_commit(self, mock_replace,
store, tmp_path):
+ store.prepare_publish_areas()
+ validate_existing = mock.create_autospec(_validate_tree)
+
+ with pytest.raises(BundleManifestError, match="I/O error"):
+ store.publish_snapshot(
+ manifest=build_bundle_version_manifest(
+ bundle_name=BUNDLE_NAME, root=_write_source(tmp_path /
"source"), backend_type="local"
+ ).manifest,
+ source_root=tmp_path / "source",
+ validate_existing=validate_existing,
+ )
+
+ validate_existing.assert_not_called()
+ assert list(_get_snapshots_dir(store).iterdir()) == []
+
+ def test_publish_snapshot_syncs_files_after_final_permissions(self, store,
tmp_path):
+ source = _write_source(tmp_path / "source")
+ (source / "dags" / "example.py").chmod(0o600)
+ synced_modes: dict[str, int] = {}
+
+ def record_mode(path):
+ temp_tree = next(parent for parent in path.parents if
parent.parent == _get_snapshots_dir(store))
+ synced_modes[path.relative_to(temp_tree).as_posix()] =
stat.S_IMODE(path.stat().st_mode)
+ filesystem_module.fsync_file(path)
+
+ with mock.patch.object(store_module, "fsync_file", autospec=True,
side_effect=record_mode):
+ _publish(store, source)
+
+ assert synced_modes == {"dags/example.py": 0o444, MANIFEST_FILE_NAME:
0o444}
+
+ @mock.patch.object(store_module.os, "scandir", autospec=True,
side_effect=_scandir_unless_unlistable)
+ def test_snapshot_permissions_fail_on_an_unlistable_directory(self,
mock_scandir, tmp_path):
+ (tmp_path / "snapshot" / "unlistable").mkdir(parents=True)
+
+ with pytest.raises(PermissionError, match="cannot list directory"):
+ store_module._set_snapshot_permissions(tmp_path / "snapshot")
+
+ @mock.patch.object(
+ store_module, "fsync_directory", autospec=True,
side_effect=store_module.fsync_directory
+ )
+ def
test_publish_snapshot_retry_syncs_a_snapshot_committed_before_a_sync_failure(
+ self, mock_fsync, store, tmp_path
+ ):
+ source = _write_source(tmp_path / "source")
+ result = build_bundle_version_manifest(bundle_name=BUNDLE_NAME,
root=source, backend_type="local")
+ store.prepare_publish_areas()
+ real_fsync_directory = mock_fsync.side_effect
+ mock_fsync.side_effect = OSError(errno.EIO, "directory sync failed")
+ with pytest.raises(BundleManifestError, match="directory sync failed"):
+ store.publish_snapshot(
+ manifest=result.manifest, source_root=source,
validate_existing=_validate_tree
+ )
+ assert store.snapshot_exists(result.version)
+ mock_fsync.side_effect = real_fsync_directory
+ mock_fsync.reset_mock()
+ validate_existing = mock.create_autospec(_validate_tree)
+
+ publication = store.publish_snapshot(
+ manifest=result.manifest, source_root=source,
validate_existing=validate_existing
+ )
+
+ assert publication.created is False
+ validate_existing.assert_called_once_with(_get_snapshots_dir(store) /
result.version)
+ mock_fsync.assert_called_once_with(_get_snapshots_dir(store))
+
+ @pytest.mark.parametrize("area", ["versions", "snapshots", "reference",
"lock"])
+ @mock.patch.object(Path, "chmod", autospec=True, side_effect=Path.chmod)
+ def test_publication_retries_directory_creation_after_chmod_failure(
+ self, mock_chmod, store, tmp_path, area
+ ):
+ source = _write_source(tmp_path / "source")
+ failed_directory = {
+ "versions": _get_snapshots_dir(store).parent,
+ "snapshots": _get_snapshots_dir(store),
+ "reference": _get_ref_path(store).parent,
+ "lock": _get_lock_path(store).parent,
+ }[area]
+ original_error = OSError(errno.EIO, "chmod failed")
+ real_chmod = mock_chmod.side_effect
+
+ def fail_chmod(path, mode, **kwargs):
+ if path == failed_directory:
+ raise original_error
+ return real_chmod(path, mode, **kwargs)
+
+ previous_umask = os.umask(0o077)
+ try:
+ mock_chmod.side_effect = fail_chmod
+ with pytest.raises(BundleManifestError, match="chmod failed") as
raised:
+ with store.publication_guard():
+ _publish(store, source)
+ assert raised.value.__cause__ is original_error
+ assert not failed_directory.exists()
+
+ mock_chmod.side_effect = real_chmod
+ with store.publication_guard():
+ result, publication = _publish(store, source)
+ store.write_ref(result.ref_payload)
+ finally:
+ os.umask(previous_umask)
+
+ assert publication.created is True
+ assert store.read_ref() == result.ref_payload
+ assert stat.S_IMODE(failed_directory.stat().st_mode) == 0o755
+
+ @pytest.mark.parametrize("other_owner", [False, True])
+ @mock.patch.object(Path, "rmdir", autospec=True, side_effect=Path.rmdir)
+ @mock.patch.object(Path, "chmod", autospec=True, side_effect=Path.chmod)
+ @mock.patch.object(Path, "mkdir", autospec=True, side_effect=Path.mkdir)
+ def test_failed_chmod_preserves_a_directory_created_concurrently(
+ self, mock_mkdir, mock_chmod, mock_rmdir, store, other_owner
+ ):
+ raced_directory = _get_snapshots_dir(store).parent
+ real_mkdir = mock_mkdir.side_effect
+ real_chmod = mock_chmod.side_effect
+
+ def create_rival_first(path, *args, **kwargs):
+ if path == raced_directory:
+ real_mkdir(path, mode=0o750)
+ real_chmod(path, 0o750)
+ return real_mkdir(path, *args, **kwargs)
+
+ def fail_chmod(path, mode, **kwargs):
+ if path == raced_directory:
+ raise PermissionError(errno.EPERM, "chmod denied")
+ return real_chmod(path, mode, **kwargs)
+
+ mock_mkdir.side_effect = create_rival_first
+ mock_chmod.side_effect = fail_chmod
+ with mock.patch.object(
+ store_module.os, "geteuid", autospec=True,
return_value=os.geteuid() + other_owner
+ ):
+ if other_owner:
+ store.prepare_publish_areas()
+ else:
+ with pytest.raises(BundleManifestError, match="chmod denied"):
+ store.prepare_publish_areas()
+
+ assert stat.S_IMODE(raced_directory.stat().st_mode) == 0o750
+ mock_rmdir.assert_not_called()
+
+ @mock.patch.object(Path, "chmod", autospec=True, side_effect=Path.chmod)
+ @mock.patch.object(Path, "mkdir", autospec=True, side_effect=Path.mkdir)
+ def test_publication_directory_creation_preserves_a_racing_file(self,
mock_mkdir, mock_chmod, store):
+ raced_file = _get_snapshots_dir(store).parent
+ real_mkdir = mock_mkdir.side_effect
+
+ def create_file_first(path, *args, **kwargs):
+ if path == raced_file:
+ path.write_text("unrelated data")
+ return real_mkdir(path, *args, **kwargs)
+
+ mock_mkdir.side_effect = create_file_first
+ with pytest.raises(BundleManifestError) as raised:
+ store.prepare_publish_areas()
+
+ assert isinstance(raised.value.__cause__, FileExistsError)
+ assert raced_file.read_text() == "unrelated data"
+ mock_chmod.assert_not_called()
+
+ @mock.patch.object(store_module, "log", autospec=True)
+ @mock.patch.object(Path, "rmdir", autospec=True,
side_effect=OSError(errno.EIO, "cleanup failed"))
+ @mock.patch.object(Path, "chmod", autospec=True,
side_effect=OSError(errno.EIO, "chmod failed"))
+ def test_failed_directory_cleanup_preserves_the_permission_error(
+ self, mock_chmod, mock_rmdir, mock_log, store
+ ):
+ with pytest.raises(BundleManifestError, match="chmod failed") as
raised:
+ store.prepare_publish_areas()
+
+ versions_directory = _get_snapshots_dir(store).parent
+ assert raised.value.__cause__ is mock_chmod.side_effect
+ mock_rmdir.assert_called_once_with(versions_directory)
+ assert versions_directory.is_dir()
+ mock_log.warning.assert_called_once_with(
+ "Could not remove publication directory %s after its setup failed;
inspect it before retrying",
+ versions_directory,
+ exc_info=True,
+ )
+
+ @pytest.mark.parametrize(
+ ("owner_offset", "fails"),
+ [pytest.param(0, True, id="owned"), pytest.param(1, False,
id="owned-by-another-publisher")],
+ )
+ @mock.patch.object(
+ store_module.os, "fchmod", autospec=True,
side_effect=OSError(errno.EIO, "fchmod failed")
+ )
+ def test_publication_guard_reports_chmod_failures_on_an_owned_lock(
+ self, mock_fchmod, store, owner_offset, fails
+ ):
+ owner = os.geteuid() + owner_offset
+ entered = False
+ with mock.patch.object(store_module.os, "geteuid", autospec=True,
return_value=owner):
+ if fails:
+ with pytest.raises(BundleManifestError, match="fchmod
failed"), store.publication_guard():
+ entered = True
+ else:
+ with store.publication_guard():
+ entered = True
+
+ assert entered is not fails
+
+ @mock.patch.object(store_module, "log", autospec=True)
+ @mock.patch.object(store_module, "flock", autospec=True,
side_effect=_fail_unlock)
+ def test_publication_guard_keeps_the_publication_error_when_unlock_fails(
+ self, mock_flock, mock_log, store
+ ):
+ durability_error =
BundleManifestRefDurabilityError(ref_locator=store.ref_locator, version="v1")
+
+ with pytest.raises(BundleManifestRefDurabilityError) as raised,
store.publication_guard():
+ raise durability_error
+
+ assert raised.value is durability_error
+ mock_log.warning.assert_called_once_with(
+ "Cannot release publication lock %s", _get_lock_path(store),
exc_info=True
+ )
+
+ @mock.patch.object(store_module, "flock", autospec=True,
side_effect=_fail_unlock)
+ def test_publication_guard_reports_a_failed_unlock(self, mock_flock,
store):
+ with (
+ pytest.raises(BundleManifestError, match="Cannot release
publication lock .*unlock failed"),
+ store.publication_guard(),
+ ):
+ pass
+
+ @mock.patch.object(store_module.shutil, "copystat", autospec=True)
+ def test_publish_and_fetch_copy_no_file_or_directory_metadata(self,
mock_copystat, store, tmp_path):
+ result, _ = _publish(store, _write_source(tmp_path / "source"))
+ store.fetch_snapshot(result.version, tmp_path / "dest",
structural_validator=_validate_tree)
+
+ mock_copystat.assert_not_called()
+
+ @mock.patch.object(
+ store_module, "fsync_directory", autospec=True,
side_effect=store_module.fsync_directory
+ )
+ def test_existing_publication_directories_are_not_synced_again(self,
mock_fsync, store):
+ store.prepare_publish_areas()
+ mock_fsync.reset_mock()
+
+ store.prepare_publish_areas()
+
+ mock_fsync.assert_not_called()
+
+ @mock.patch.object(
+ store_module, "fsync_directory", autospec=True,
side_effect=store_module.fsync_directory
+ )
+ def
test_first_call_syncs_directories_left_by_an_interrupted_publisher(self,
mock_fsync, store):
+ # Created by a publisher that was killed before syncing them into
their parents.
+ _get_snapshots_dir(store).mkdir(parents=True)
+ _get_ref_path(store).parent.mkdir(parents=True)
+
+ store.prepare_publish_areas()
+
+ assert {call.args[0] for call in mock_fsync.call_args_list} == {
+ _get_snapshots_dir(store),
+ _get_snapshots_dir(store).parent,
+ _get_ref_path(store).parent,
+ _get_ref_path(store).parent.parent,
+ store.published_root,
+ }
+
+ @pytest.mark.parametrize("area", ["versions", "refs"])
+ @mock.patch.object(
+ store_module, "fsync_directory", autospec=True,
side_effect=store_module.fsync_directory
+ )
+ def
test_failed_sync_of_existing_directories_is_retried_on_the_same_store(self,
mock_fsync, store, area):
+ _get_snapshots_dir(store).mkdir(parents=True)
+ _get_ref_path(store).parent.mkdir(parents=True)
+ interrupted_directory = store.published_root / area
+ real_fsync = mock_fsync.side_effect
+
+ def fail_ancestor_sync(path):
+ if path == interrupted_directory:
+ raise OSError("sync interrupted")
+ return real_fsync(path)
+
+ mock_fsync.side_effect = fail_ancestor_sync
+ with pytest.raises(BundleManifestError, match="sync interrupted"):
+ store.prepare_publish_areas()
+
+ mock_fsync.side_effect = real_fsync
+ mock_fsync.reset_mock()
+
+ store.prepare_publish_areas()
+
+ mock_fsync.assert_any_call(interrupted_directory)
+ mock_fsync.assert_any_call(store.published_root)
+
+ def test_publish_snapshot_rejects_file_at_version_path(self, store,
tmp_path):
+ source = _write_source(tmp_path / "source")
+ result = build_bundle_version_manifest(bundle_name=BUNDLE_NAME,
root=source, backend_type="local")
+ store.prepare_publish_areas()
+ (_get_snapshots_dir(store) / result.version).write_text("not a
snapshot")
+
+ with pytest.raises(BundleManifestError, match="is not a directory"):
+ store.publish_snapshot(
+ manifest=result.manifest,
+ source_root=source,
+ validate_existing=lambda tree: None,
+ )
+
+ @pytest.mark.parametrize(
+ ("previous_payload", "payload", "expected_version"),
+ [
+ pytest.param(None, {"version": "v2"}, "v2",
id="first-publication"),
+ pytest.param({"version": "v1"}, {"version": "v2"}, "v2",
id="replacement"),
+ pytest.param({"version": "v1"}, {"schema_version": 1}, None,
id="missing-version"),
+ pytest.param({"version": "v1"}, {"version": 2}, None,
id="non-string-version"),
+ ],
+ )
+ @mock.patch.object(
+ store_module, "fsync_directory", autospec=True,
side_effect=store_module.fsync_directory
+ )
+ def test_write_ref_reports_durability_failure_after_replacement(
+ self, mock_fsync, store, previous_payload, payload, expected_version
+ ):
+ if previous_payload is not None:
+ store.write_ref(previous_payload)
+ original_error = OSError(errno.EIO, "reference directory sync failed")
+ real_fsync_directory = mock_fsync.side_effect
+
+ def fail_after_replacement(path):
+ if (
+ path == _get_ref_path(store).parent
+ and _get_ref_path(store).exists()
+ and store.read_ref() == payload
+ ):
+ raise original_error
+ real_fsync_directory(path)
+
+ mock_fsync.side_effect = fail_after_replacement
+
+ with pytest.raises(
+ BundleManifestRefDurabilityError,
+ match="may have been replaced, but durability could not be
confirmed",
+ ) as raised:
+ store.write_ref(payload)
+
+ assert isinstance(raised.value, BundleManifestError)
+ assert raised.value.ref_locator == store.ref_locator
+ assert raised.value.version == expected_version
+ assert raised.value.__cause__ is original_error
+ assert store.ref_locator in str(raised.value)
+ assert store.read_ref() == payload
+ assert list(_get_ref_path(store).parent.iterdir()) ==
[_get_ref_path(store)]
+
+ @pytest.mark.parametrize(
+ ("existing_reference", "cleanup_fails"), [(False, False), (True,
False), (True, True)]
+ )
+ @mock.patch.object(Path, "unlink", autospec=True, side_effect=Path.unlink)
+ @mock.patch.object(store_module.os, "replace", autospec=True,
side_effect=store_module.os.replace)
+ def test_write_ref_chains_original_error_when_replace_fails(
+ self, mock_replace, mock_unlink, store, existing_reference,
cleanup_fails
+ ):
+ store.prepare_publish_areas()
+ if existing_reference:
+ store.write_ref({"version": "v1"})
+ original_error = OSError(errno.EIO, "replace failed")
+ mock_replace.side_effect = original_error
+ if cleanup_fails:
+ mock_unlink.side_effect = PermissionError("cleanup failed")
+
+ with pytest.raises(BundleManifestError, match="replace failed") as
raised:
+ store.write_ref({"version": "v2"})
+
+ assert raised.value.__cause__ is original_error
+ if existing_reference:
+ assert store.read_ref() == {"version": "v1"}
+ else:
+ assert not _get_ref_path(store).exists()
+ temp_path = mock_replace.call_args.args[0]
+ mock_unlink.assert_called_once_with(temp_path, missing_ok=True)
+ assert temp_path.exists() is cleanup_fails
+ assert set(_get_ref_path(store).parent.iterdir()) == (
+ ({_get_ref_path(store)} if existing_reference else set())
+ | ({temp_path} if cleanup_fails else set())
+ )
+
+ @mock.patch.object(store_module.os, "replace", autospec=True,
side_effect=_replace_then_fail)
+ @mock.patch.object(
+ store_module, "fsync_directory", autospec=True,
side_effect=store_module.fsync_directory
+ )
+ def test_write_ref_accepts_a_rename_that_reported_failure(self,
mock_fsync, mock_replace, store):
+ store.prepare_publish_areas()
+
+ store.write_ref({"version": "v1"})
+
+ mock_replace.assert_called_once()
+ assert store.read_ref() == {"version": "v1"}
+ assert mock_fsync.call_args == mock.call(_get_ref_path(store).parent)
+ assert list(_get_ref_path(store).parent.iterdir()) ==
[_get_ref_path(store)]
+
+ @pytest.mark.parametrize(
+ "previous_payload",
+ [
+ pytest.param(None, id="first-publication"),
+ pytest.param({"version": "v1"}, id="replacement"),
+ ],
+ )
+ @mock.patch.object(store_module.os, "replace", autospec=True,
side_effect=_real_replace)
+ def test_write_ref_reports_a_swept_temp_file_as_durability_error(
+ self, mock_replace, store, previous_payload
+ ):
+ store.prepare_publish_areas()
+ if previous_payload is not None:
+ store.write_ref(previous_payload)
+ mock_replace.side_effect = _sweep_then_fail
+
+ with pytest.raises(BundleManifestRefDurabilityError) as raised:
+ store.write_ref({"version": "v2"})
+
+ assert raised.value.version == "v2"
+ assert isinstance(raised.value.__cause__, FileNotFoundError)
+ assert raised.value.__cause__.filename ==
os.fspath(mock_replace.call_args.args[0])
+ if previous_payload is None:
+ assert list(_get_ref_path(store).parent.iterdir()) == []
+ else:
+ assert store.read_ref() == previous_payload
+ assert list(_get_ref_path(store).parent.iterdir()) ==
[_get_ref_path(store)]
+
+ @mock.patch.object(
+ store_module.os, "replace", autospec=True,
side_effect=OSError(errno.EIO, "rename reply lost")
+ )
+ def
test_write_ref_reports_an_unverifiable_rename_as_durability_error(self,
mock_replace, store):
+ store.prepare_publish_areas()
+ lstat_error = OSError(errno.EIO, "cannot stat temporary reference")
+ real_lstat = Path.lstat
+
+ def fail_temp_lstat(path):
+ if path == mock_replace.call_args.args[0]:
+ raise lstat_error
+ return real_lstat(path)
+
+ with (
+ mock.patch.object(Path, "lstat", autospec=True,
side_effect=fail_temp_lstat),
+ pytest.raises(BundleManifestRefDurabilityError) as raised,
+ ):
+ store.write_ref({"version": "v1"})
+
+ assert raised.value.version == "v1"
+ assert raised.value.__cause__ is lstat_error
+
+ def
test_publication_guard_removes_read_only_orphan_and_keeps_other_hidden_dirs(self,
store):
+ store.prepare_publish_areas()
+ orphan = _get_snapshots_dir(store) / f".sha256-{'1' * 64}.abandoned"
+ (orphan / "dags").mkdir(parents=True)
+ (orphan / "dags" / "partial.py").write_text("print('partial')")
+ store_module._set_snapshot_permissions(orphan)
+ unrelated_hidden_dir = _get_snapshots_dir(store) /
".not-a-publisher-temp"
+ unrelated_hidden_dir.mkdir()
+
+ with store.publication_guard():
+ pass
+
+ assert not orphan.exists()
+ assert unrelated_hidden_dir.exists()
+
+ @mock.patch.object(
+ filesystem_module,
+ "remove_bundle_tree_forcefully",
+ autospec=True,
+ side_effect=PermissionError("denied"),
+ )
+ def test_publication_guard_survives_undeletable_orphan(self, mock_remove,
store):
+ store.prepare_publish_areas()
+ orphan = _get_snapshots_dir(store) / f".sha256-{'1' * 64}.abandoned"
+ orphan.mkdir()
+
+ with store.publication_guard():
+ pass
+
+ mock_remove.assert_called_once_with(orphan)
+ assert orphan.exists()
+
+
+_STORAGE_FAULTS = [
+ ("prepare_publish_areas", store_module, "_ensure_public_dir"),
+ ("snapshot_exists", Path, "stat"),
+ ("fetch_snapshot", store_module.shutil, "copy"),
+ ("publish_snapshot", store_module, "fsync_tree_directories"),
+ ("publication_guard", store_module, "remove_orphaned_temp_snapshots"),
+ ("read_ref", Path, "read_text"),
+ ("write_ref", store_module.os, "replace"),
+]
+_STORAGE_ERRORS = [
+ ("missing", FileNotFoundError(errno.ENOENT, "gone"),
BundleManifestNotFoundError),
+ ("denied", PermissionError(errno.EACCES, "denied"), BundleManifestError),
+]
+
+
+class TestFilesystemStorageErrors:
+ @pytest.mark.parametrize(
+ ("operation", "patch_owner", "patch_name", "error", "expected"),
+ [
+ pytest.param(operation, owner, name, error, expected,
id=f"{operation}-{error_id}")
+ for operation, owner, name in _STORAGE_FAULTS
+ for error_id, error, expected in _STORAGE_ERRORS
+ # A missing reference is covered by
test_read_ref_missing_raises_not_found, and a
+ # missing snapshot is not an error for snapshot_exists.
+ if (operation, error_id) not in {("read_ref", "missing"),
("snapshot_exists", "missing")}
+ ],
+ )
+ def test_storage_oserror_is_reported_as_bundle_error(
+ self, store, tmp_path, operation, patch_owner, patch_name, error,
expected
+ ):
+ source = _write_source(tmp_path / "source")
+ store.prepare_publish_areas()
+ if operation == "fetch_snapshot":
+ version = _publish(store, source)[0].version
+ operations = {
+ "prepare_publish_areas": store.prepare_publish_areas,
+ "snapshot_exists": partial(store.snapshot_exists, "sha256-" + "0"
* 64),
+ "fetch_snapshot": lambda: store.fetch_snapshot(
+ version, tmp_path / "dest", structural_validator=lambda tree:
None
+ ),
+ "publish_snapshot": partial(_publish, store, source),
+ "publication_guard": lambda: store.publication_guard().__enter__(),
+ "read_ref": store.read_ref,
+ "write_ref": partial(store.write_ref, {"version": "v1"}),
+ }
+
+ with (
+ mock.patch.object(patch_owner, patch_name, autospec=True,
side_effect=error),
+ pytest.raises(expected) as raised,
+ ):
+ operations[operation]()
+
+ assert not isinstance(raised.value, BundleManifestSourceChangedError)
+ assert raised.value.__cause__ is error
+
+ @pytest.mark.parametrize(
+ ("failing_side", "error", "expected"),
+ [
+ pytest.param(
+ "source",
+ FileNotFoundError(errno.ENOENT, "gone"),
+ BundleManifestSourceChangedError,
+ id="source-gone",
+ ),
+ pytest.param(
+ "source", PermissionError(errno.EACCES, "denied"),
BundleManifestError, id="source-denied"
+ ),
+ pytest.param(
+ "published",
+ FileNotFoundError(errno.ENOENT, "gone"),
+ BundleManifestNotFoundError,
+ id="published-gone",
+ ),
+ ],
+ )
+ def test_publish_snapshot_attributes_copy_failures(self, store, tmp_path,
failing_side, error, expected):
+ source = _write_source(tmp_path / "source")
+
+ def fail_copy(src, dst, **kwargs):
+ error.filename = os.fspath(src if failing_side == "source" else
dst)
+ raise error
+
+ with (
+ mock.patch.object(store_module.shutil, "copy", autospec=True,
side_effect=fail_copy),
+ pytest.raises(BundleManifestError) as raised,
+ ):
+ _publish(store, source)
+
+ assert type(raised.value) is expected
+ assert list(_get_snapshots_dir(store).iterdir()) == []
+
+ @mock.patch.object(store_module.shutil, "copy", autospec=True,
side_effect=_fail_fast_copy)
+ def
test_publish_snapshot_reports_fast_copy_failures_as_storage_errors(self,
mock_copy, store, tmp_path):
+ with pytest.raises(BundleManifestError, match="Cannot publish snapshot
.*No space left") as raised:
+ _publish(store, _write_source(tmp_path / "source"))
+
+ assert type(raised.value) is BundleManifestError
+ assert list(_get_snapshots_dir(store).iterdir()) == []
+
+ @pytest.mark.parametrize("operation", ["snapshot_exists",
"fetch_snapshot", "publish_snapshot"])
+ @mock.patch.object(Path, "exists", autospec=True,
side_effect=_exists_as_in_python_314)
+ @mock.patch.object(Path, "is_dir", autospec=True,
side_effect=_is_dir_as_in_python_314)
+ def test_snapshot_storage_errors_are_not_reported_as_missing(
+ self, mock_is_dir, mock_exists, store, tmp_path, operation
+ ):
+ source = _write_source(tmp_path / "source")
+ result, _ = _publish(store, source)
+ version_path = os.fspath(_get_snapshots_dir(store) / result.version)
+ stat_error = PermissionError(errno.EACCES, "denied")
+ real_stat = os.stat
+
+ def deny_snapshot(path, *args, **kwargs):
+ if os.fspath(path) == version_path:
+ raise stat_error
+ return real_stat(path, *args, **kwargs)
+
+ operations = {
+ "snapshot_exists": partial(store.snapshot_exists, result.version),
+ "fetch_snapshot": partial(
+ store.fetch_snapshot, result.version, tmp_path / "dest",
structural_validator=_validate_tree
+ ),
+ "publish_snapshot": partial(
+ store.publish_snapshot,
+ manifest=result.manifest,
+ source_root=source,
+ validate_existing=_validate_tree,
+ ),
+ }
+ with (
+ mock.patch.object(os, "stat", autospec=True,
side_effect=deny_snapshot),
+ pytest.raises(BundleManifestError) as raised,
+ ):
+ operations[operation]()
+
+ assert type(raised.value) is BundleManifestError
+ assert raised.value.__cause__ is stat_error
+
+ @mock.patch.object(Path, "read_text", autospec=True,
side_effect=PermissionError(errno.EACCES, "denied"))
+ def test_read_json_file_reports_unreadable_document(self, mock_read_text,
tmp_path):
+ with pytest.raises(BundleManifestError, match="Cannot read") as raised:
+ read_json_file(tmp_path / "doc.json", missing_message="missing",
invalid_message="invalid")
+
+ assert raised.value.__cause__ is mock_read_text.side_effect